garbage collection for cross-namespace workloads and traits in application layer (#1421)

* add logic for gc cross ns resources

WIP regenerate api and charts

rewrite handle resourceTracker logic

WIP add e2e test logic

add more e2e  test

* WIP refactor sevral funs name and add comments for code

* WIP add more corener case e2e test

* add unit test and fix bug

* WIP refactor handleResouceTracker  logic

* refactor func name aglin with others and add comments

* make generate to fix check-diff ci error

* change resourceTracker status as a subresource

apis/core.oam.dev/v1beta1/core_types.go

@@ -20,6 +20,7 @@ import (
 	runtimev1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
 )
@@ -236,12 +237,41 @@ type ScopeDefinitionList struct {
 }
 
 // +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
 
 // An ResourceTracker represents a tracker for track cross namespace resources
 // +kubebuilder:resource:scope=Cluster,categories={oam}
 type ResourceTracker struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Status ResourceTrackerStatus `json:"status,omitempty"`
+}
+
+// ResourceTrackerStatus define the status of resourceTracker
+type ResourceTrackerStatus struct {
+	TrackedResources []TypedReference `json:"trackedResources,omitempty"`
+}
+
+// A TypedReference refers to an object by Name, Kind, and APIVersion. It is
+// commonly used to reference across-namespace objects
+type TypedReference struct {
+	// APIVersion of the referenced object.
+	APIVersion string `json:"apiVersion"`
+
+	// Kind of the referenced object.
+	Kind string `json:"kind"`
+
+	// Name of the referenced object.
+	Name string `json:"name"`
+
+	// Namespace of the objects outside the application namespace.
+	// +optional
+	Namespace string `json:"namespace,omitempty"`
+
+	// UID of the referenced object.
+	// +optional
+	UID types.UID `json:"uid,omitempty"`
 }
 
 // +kubebuilder:object:root=true

apis/core.oam.dev/v1beta1/zz_generated.deepcopy.go

@@ -862,6 +862,7 @@ func (in *ResourceTracker) DeepCopyInto(out *ResourceTracker) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Status.DeepCopyInto(&out.Status)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTracker.
@@ -914,6 +915,26 @@ func (in *ResourceTrackerList) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ResourceTrackerStatus) DeepCopyInto(out *ResourceTrackerStatus) {
+	*out = *in
+	if in.TrackedResources != nil {
+		in, out := &in.TrackedResources, &out.TrackedResources
+		*out = make([]TypedReference, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTrackerStatus.
+func (in *ResourceTrackerStatus) DeepCopy() *ResourceTrackerStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ResourceTrackerStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ScopeDefinition) DeepCopyInto(out *ScopeDefinition) {
 	*out = *in
@@ -1141,6 +1162,21 @@ func (in *TraitDefinitionStatus) DeepCopy() *TraitDefinitionStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TypedReference) DeepCopyInto(out *TypedReference) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TypedReference.
+func (in *TypedReference) DeepCopy() *TypedReference {
+	if in == nil {
+		return nil
+	}
+	out := new(TypedReference)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *URIMatch) DeepCopyInto(out *URIMatch) {
 	*out = *in

charts/vela-core/crds/core.oam.dev_resourcetrackers.yaml

@@ -30,9 +30,40 @@ spec:
             type: string
           metadata:
             type: object
+          status:
+            description: ResourceTrackerStatus define the status of resourceTracker
+            properties:
+              trackedResources:
+                items:
+                  description: A TypedReference refers to an object by Name, Kind, and APIVersion. It is commonly used to reference across-namespace objects
+                  properties:
+                    apiVersion:
+                      description: APIVersion of the referenced object.
+                      type: string
+                    kind:
+                      description: Kind of the referenced object.
+                      type: string
+                    name:
+                      description: Name of the referenced object.
+                      type: string
+                    namespace:
+                      description: Namespace of the objects outside the application namespace.
+                      type: string
+                    uid:
+                      description: UID of the referenced object.
+                      type: string
+                  required:
+                  - apiVersion
+                  - kind
+                  - name
+                  type: object
+                type: array
+            type: object
         type: object
     served: true
     storage: true
+    subresources:
+      status: {}
 status:
   acceptedNames:
     kind: ""

docs/en/cue/cross-namespace-resource.md

@@ -50,7 +50,4 @@ spec:
         			}}}
         }
 ```
-## Limitations
-If you update definition by changing the `metadata.namespace` field. KubeVela will create new resources in the new namespace but not delete old resources.
-We wil fix the limitation in the near future.
 

legacy/charts/vela-core-legacy/crds/core.oam.dev_resourcetrackers.yaml

@@ -16,6 +16,8 @@ spec:
     plural: resourcetrackers
     singular: resourcetracker
   scope: Cluster
+  subresources:
+    status: {}
   validation:
     openAPIV3Schema:
       description: An ResourceTracker represents a tracker for track cross namespace resources
@@ -28,6 +30,35 @@ spec:
           type: string
         metadata:
           type: object
+        status:
+          description: ResourceTrackerStatus define the status of resourceTracker
+          properties:
+            trackedResources:
+              items:
+                description: A TypedReference refers to an object by Name, Kind, and APIVersion. It is commonly used to reference across-namespace objects
+                properties:
+                  apiVersion:
+                    description: APIVersion of the referenced object.
+                    type: string
+                  kind:
+                    description: Kind of the referenced object.
+                    type: string
+                  name:
+                    description: Name of the referenced object.
+                    type: string
+                  namespace:
+                    description: Namespace of the objects outside the application namespace.
+                    type: string
+                  uid:
+                    description: UID of the referenced object.
+                    type: string
+                required:
+                - apiVersion
+                - kind
+                - name
+                type: object
+              type: array
+          type: object
       type: object
   version: v1beta1
   versions:

pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go

@@ -145,6 +145,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	err = handler.handleResourceTracker(ctx, comps, ac)
 	if err != nil {
 		applog.Error(err, "[Handle resourceTracker]")
+		app.Status.SetConditions(errorCondition("Handle resourceTracker", err))
 		return handler.handleErr(err)
 	}
 
@@ -180,6 +181,12 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	app.Status.Services = appCompStatus
 	app.Status.SetConditions(readyCondition("HealthCheck"))
 	app.Status.Phase = common.ApplicationRunning
+	err = handler.garbageCollection(ctx)
+	if err != nil {
+		applog.Error(err, "[Garbage collection]")
+		app.Status.SetConditions(errorCondition("GarbageCollection", err))
+		return handler.handleErr(err)
+	}
 	// Gather status of components
 	var refComps []v1alpha1.TypedReference
 	for _, comp := range comps {

pkg/controller/core.oam.dev/v1alpha2/application/application_finalizer_test.go

@@ -166,6 +166,88 @@ var _ = Describe("Test application controller finalizer logic", func() {
 		checkRt := new(v1beta1.ResourceTracker)
 		Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), checkRt)).Should(util.NotFoundMatcher{})
 	})
+
+	It("Test cross namespace workload, then update the app to change the namespace", func() {
+		appName := "app-3"
+		appKey := types.NamespacedName{Namespace: namespace, Name: appName}
+		app := getApp(appName, namespace, "cross-worker")
+		Expect(k8sClient.Create(ctx, app)).Should(BeNil())
+
+		By("Create a cross workload app")
+		_, err := reconciler.Reconcile(ctrl.Request{NamespacedName: appKey})
+		Expect(err).Should(BeNil())
+		checkApp := &v1beta1.Application{}
+		Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
+		Expect(checkApp.Status.Phase).Should(Equal(common.ApplicationRunning))
+		Expect(len(checkApp.Finalizers)).Should(BeEquivalentTo(0))
+		rt := &v1beta1.ResourceTracker{}
+		Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), rt)).Should(BeNil())
+		_, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey})
+		Expect(err).Should(BeNil())
+		checkApp = new(v1beta1.Application)
+		Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
+		Expect(len(checkApp.Finalizers)).Should(BeEquivalentTo(1))
+		Expect(checkApp.Finalizers[0]).Should(BeEquivalentTo(resourceTrackerFinalizer))
+		Expect(checkApp.Status.ResourceTracker.UID).Should(BeEquivalentTo(rt.UID))
+		Expect(len(rt.Status.TrackedResources)).Should(BeEquivalentTo(1))
+		By("Update the app, set type to normal-worker")
+		checkApp.Spec.Components[0].Type = "normal-worker"
+		Expect(k8sClient.Update(ctx, checkApp)).Should(BeNil())
+		_, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey})
+		Expect(err).Should(BeNil())
+		checkApp = new(v1beta1.Application)
+		Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
+		Expect(checkApp.Status.ResourceTracker).Should(BeNil())
+		Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), rt)).Should(util.NotFoundMatcher{})
+		Expect(k8sClient.Delete(ctx, checkApp)).Should(BeNil())
+		_, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey})
+		Expect(err).Should(BeNil())
+	})
+
+	It("Test cross namespace workload and trait, then update the app to delete trait ", func() {
+		appName := "app-4"
+		appKey := types.NamespacedName{Namespace: namespace, Name: appName}
+		app := getApp(appName, namespace, "cross-worker")
+		app.Spec.Components[0].Traits = []v1beta1.ApplicationTrait{
+			{
+				Type:       "cross-scaler",
+				Properties: runtime.RawExtension{Raw: []byte(`{"replicas": 1}`)},
+			},
+		}
+		Expect(k8sClient.Create(ctx, app)).Should(BeNil())
+		By("Create a cross workload trait app")
+		_, err := reconciler.Reconcile(ctrl.Request{NamespacedName: appKey})
+		Expect(err).Should(BeNil())
+		checkApp := &v1beta1.Application{}
+		Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
+		Expect(checkApp.Status.Phase).Should(Equal(common.ApplicationRunning))
+		Expect(len(checkApp.Finalizers)).Should(BeEquivalentTo(0))
+		rt := &v1beta1.ResourceTracker{}
+		Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), rt)).Should(BeNil())
+		_, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey})
+		Expect(err).Should(BeNil())
+		checkApp = new(v1beta1.Application)
+		Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
+		Expect(len(checkApp.Finalizers)).Should(BeEquivalentTo(1))
+		Expect(checkApp.Finalizers[0]).Should(BeEquivalentTo(resourceTrackerFinalizer))
+		Expect(checkApp.Status.ResourceTracker.UID).Should(BeEquivalentTo(rt.UID))
+		Expect(len(rt.Status.TrackedResources)).Should(BeEquivalentTo(2))
+		By("Update the app, set type to normal-worker")
+		checkApp.Spec.Components[0].Traits = nil
+		Expect(k8sClient.Update(ctx, checkApp)).Should(BeNil())
+		_, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey})
+		Expect(err).Should(BeNil())
+		rt = &v1beta1.ResourceTracker{}
+		checkApp = new(v1beta1.Application)
+		Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
+		Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), rt)).Should(BeNil())
+		Expect(checkApp.Status.ResourceTracker.UID).Should(BeEquivalentTo(rt.UID))
+		Expect(len(rt.Status.TrackedResources)).Should(BeEquivalentTo(1))
+		Expect(k8sClient.Delete(ctx, checkApp)).Should(BeNil())
+		_, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey})
+		Expect(err).Should(BeNil())
+		Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), rt)).Should(util.NotFoundMatcher{})
+	})
 })
 
 var _ = Describe("Test finalizer related func", func() {
@@ -186,46 +268,6 @@ var _ = Describe("Test finalizer related func", func() {
 		By("[TEST] Clean up resources after an integration test")
 	})
 
-	It("Test getResourceTrackerAndOwnReference func", func() {
-		app := getApp("app-1", namespace, "worker")
-		handler = appHandler{
-			r:      reconciler,
-			app:    app,
-			logger: reconciler.Log.WithValues("application", "finalizer-func-test"),
-		}
-		checkRt := new(v1beta1.ResourceTracker)
-		Expect(k8sClient.Get(ctx, getTrackerKey(namespace, app.Name), checkRt)).Should(util.NotFoundMatcher{})
-		rt, owner, err := handler.getResourceTrackerAndOwnReference(ctx)
-		Expect(err).Should(BeNil())
-		Expect(rt.UID).Should(BeEquivalentTo(owner.UID))
-		Expect(owner.Kind).Should(BeEquivalentTo(v1beta1.ResourceTrackerKind))
-		checkRt = new(v1beta1.ResourceTracker)
-		Expect(k8sClient.Get(ctx, getTrackerKey(namespace, app.Name), checkRt)).Should(BeNil())
-		Expect(checkRt.UID).Should(BeEquivalentTo(rt.UID))
-		Expect(k8sClient.Delete(ctx, checkRt)).Should(BeNil())
-	})
-
-	It("Test getResourceTrackerAndOwnReference func with already exsit resourceTracker", func() {
-		app := getApp("app-2", namespace, "worker")
-		handler = appHandler{
-			r:      reconciler,
-			app:    app,
-			logger: reconciler.Log.WithValues("application", "finalizer-func-test"),
-		}
-		rt := &v1beta1.ResourceTracker{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: namespace + "-" + app.GetName(),
-			},
-		}
-		Expect(k8sClient.Create(ctx, rt)).Should(BeNil())
-		checkRt, owner, err := handler.getResourceTrackerAndOwnReference(ctx)
-		Expect(err).Should(BeNil())
-		Expect(rt.UID).Should(BeEquivalentTo(checkRt.UID))
-		Expect(owner.Kind).Should(BeEquivalentTo(v1beta1.ResourceTrackerKind))
-		Expect(checkRt.UID).Should(BeEquivalentTo(owner.UID))
-		Expect(k8sClient.Delete(ctx, checkRt)).Should(BeNil())
-	})
-
 	It("Test finalizeResourceTracker func with need update ", func() {
 		app := getApp("app-3", namespace, "worker")
 		rt := &v1beta1.ResourceTracker{

pkg/controller/core.oam.dev/v1alpha2/application/apply.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"strings"
 	"time"
 
 	runtimev1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1"
@@ -31,6 +32,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	ctypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
@@ -41,6 +43,7 @@ import (
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2"
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
 	"github.com/oam-dev/kubevela/pkg/appfile"
+	"github.com/oam-dev/kubevela/pkg/controller/core.oam.dev/v1alpha2/applicationconfiguration"
 	"github.com/oam-dev/kubevela/pkg/controller/utils"
 	"github.com/oam-dev/kubevela/pkg/dsl/process"
 	"github.com/oam-dev/kubevela/pkg/oam"
@@ -67,13 +70,15 @@ func readyCondition(tpy string) runtimev1alpha1.Condition {
 }
 
 type appHandler struct {
-	r             *Reconciler
-	app           *v1beta1.Application
-	appfile       *appfile.Appfile
-	logger        logr.Logger
-	inplace       bool
-	isNewRevision bool
-	revisionHash  string
+	r                        *Reconciler
+	app                      *v1beta1.Application
+	appfile                  *appfile.Appfile
+	logger                   logr.Logger
+	inplace                  bool
+	isNewRevision            bool
+	revisionHash             string
+	acrossNamespaceResources []v1beta1.TypedReference
+	resourceTracker          *v1beta1.ResourceTracker
 }
 
 // setInplace will mark if the application should upgrade the workload within the same instance(name never changed)
@@ -108,6 +113,12 @@ func (h *appHandler) apply(ctx context.Context, appRev *v1beta1.ApplicationRevis
 		Controller: pointer.BoolPtr(true),
 	}}
 
+	if _, exist := h.app.GetAnnotations()[oam.AnnotationAppRollout]; !exist && h.app.Spec.RolloutPlan == nil {
+		h.setInplace(true)
+	} else {
+		h.setInplace(false)
+	}
+
 	// don't create components and AC if revision-only annotation is set
 	if ac.Annotations[oam.AnnotationAppRevisionOnly] == "true" {
 		h.FinalizeAppRevision(appRev, ac, comps)
@@ -116,12 +127,21 @@ func (h *appHandler) apply(ctx context.Context, appRev *v1beta1.ApplicationRevis
 
 	for _, comp := range comps {
 		comp.SetOwnerReferences(owners)
+		needTracker, err := h.checkAndSetResourceTracker(&comp.Spec.Workload)
+		if err != nil {
+			return err
+		}
 		newComp := comp.DeepCopy()
 		// newComp will be updated and return the revision name instead of the component name
 		revisionName, err := h.createOrUpdateComponent(ctx, newComp)
 		if err != nil {
 			return err
 		}
+		if needTracker {
+			if err := h.recodeTrackedWorkload(comp, revisionName); err != nil {
+				return err
+			}
+		}
 		// find the ACC that contains this component
 		for i := 0; i < len(ac.Spec.Components); i++ {
 			// update the AC using the component revision instead of component name
@@ -129,6 +149,9 @@ func (h *appHandler) apply(ctx context.Context, appRev *v1beta1.ApplicationRevis
 			if ac.Spec.Components[i].ComponentName == newComp.Name {
 				ac.Spec.Components[i].RevisionName = revisionName
 				ac.Spec.Components[i].ComponentName = ""
+				if err := h.checkResourceTrackerForTrait(ctx, ac.Spec.Components[i], newComp.Name); err != nil {
+					return err
+				}
 			}
 		}
 		if comp.Spec.Helm != nil {
@@ -148,11 +171,9 @@ func (h *appHandler) apply(ctx context.Context, appRev *v1beta1.ApplicationRevis
 	// the rollout will create AppContext which will launch the real K8s resources.
 	// Otherwise, we should create/update the appContext here when there if no rollout controller to take care of new versions
 	// In this case, the workload should update with the annotation `app.oam.dev/inplace-upgrade=true`
-	if _, exist := h.app.GetAnnotations()[oam.AnnotationAppRollout]; !exist && h.app.Spec.RolloutPlan == nil {
-		h.setInplace(true)
+	if h.inplace {
 		return h.createOrUpdateAppContext(ctx, owners)
 	}
-	h.setInplace(false)
 
 	return nil
 }
@@ -399,90 +420,30 @@ func (h *appHandler) applyHelmModuleResources(ctx context.Context, comp *v1alpha
 	return nil
 }
 
-// handleResourceTracker check the namesapce of  all components and traits, if the namespace is different with application, the tracker will own them
-func (h *appHandler) handleResourceTracker(ctx context.Context, components []*v1alpha2.Component, ac *v1alpha2.ApplicationConfiguration) error {
-	ref := new(metav1.OwnerReference)
-	// resourceTracker is cache for resourceTracker, avoid get from k8s every time
-	resourceTracker := new(v1beta1.ResourceTracker)
+// checkAndSetResourceTracker check if resource's namespace is different with application, if yes set resourceTracker as
+// resource's ownerReference
+func (h *appHandler) checkAndSetResourceTracker(resource *runtime.RawExtension) (bool, error) {
 	needTracker := false
-	for i, c := range components {
-		u, err := oamutil.RawExtension2Unstructured(&c.Spec.Workload)
-		if err != nil {
-			return err
-		}
-		if checkResourceDiffWithApp(u, h.app.Namespace) {
-			needTracker = true
-			if len(resourceTracker.Name) == 0 {
-				resourceTracker, ref, err = h.getResourceTrackerAndOwnReference(ctx)
-				if err != nil {
-					return err
-				}
-			}
-			u.SetOwnerReferences([]metav1.OwnerReference{*ref})
-			raw := oamutil.Object2RawExtension(u)
-			components[i].Spec.Workload = raw
-		}
+	u, err := oamutil.RawExtension2Unstructured(resource)
+	if err != nil {
+		return false, err
 	}
-	for _, acComponent := range ac.Spec.Components {
-		for i, t := range acComponent.Traits {
-			u, err := oamutil.RawExtension2Unstructured(&t.Trait)
-			if err != nil {
-				return err
-			}
-			if checkResourceDiffWithApp(u, h.app.Namespace) {
-				needTracker = true
-				if len(resourceTracker.Name) == 0 {
-					resourceTracker, ref, err = h.getResourceTrackerAndOwnReference(ctx)
-					if err != nil {
-						return err
-					}
-				}
-				u.SetOwnerReferences([]metav1.OwnerReference{*ref})
-				raw := oamutil.Object2RawExtension(u)
-				acComponent.Traits[i].Trait = raw
-			}
-		}
+	if checkResourceDiffWithApp(u, h.app.Namespace) {
+		needTracker = true
+		ref := h.genResourceTrackerOwnerReference()
+		// set resourceTracker as the ownerReference of workload/trait
+		u.SetOwnerReferences([]metav1.OwnerReference{*ref})
+		raw := oamutil.Object2RawExtension(u)
+		*resource = raw
+		return needTracker, nil
 	}
-	if !needTracker {
-		h.app.Status.ResourceTracker = nil
-		// check weather related resourceTracker is existed, if yes delete it
-		err := h.r.Get(ctx, ctypes.NamespacedName{Name: h.generateResourceTrackerName()}, resourceTracker)
-		if err == nil {
-			return h.r.Delete(ctx, resourceTracker)
-		}
-		if !apierrors.IsNotFound(err) {
-			return err
-		}
-		return nil
-	}
-	h.app.Status.ResourceTracker = &runtimev1alpha1.TypedReference{
-		Name:       resourceTracker.Name,
-		Kind:       v1beta1.ResourceTrackerGroupKind,
-		APIVersion: v1beta1.ResourceTrackerKindAPIVersion,
-		UID:        resourceTracker.UID}
-
-	return nil
+	return needTracker, nil
 }
 
-func (h *appHandler) getResourceTrackerAndOwnReference(ctx context.Context) (*v1beta1.ResourceTracker, *metav1.OwnerReference, error) {
-	resourceTracker := new(v1beta1.ResourceTracker)
-	key := ctypes.NamespacedName{Name: h.generateResourceTrackerName()}
-	err := h.r.Get(ctx, key, resourceTracker)
-	if err != nil {
-		if apierrors.IsNotFound(err) {
-			resourceTracker = &v1beta1.ResourceTracker{
-				ObjectMeta: metav1.ObjectMeta{
-					Name: h.generateResourceTrackerName(),
-				},
-			}
-			if err = h.r.Client.Create(ctx, resourceTracker); err != nil {
-				return nil, nil, err
-			}
-			return resourceTracker, metav1.NewControllerRef(resourceTracker, v1beta1.ResourceTrackerKindVersionKind), nil
-		}
-		return nil, nil, err
-	}
-	return resourceTracker, metav1.NewControllerRef(resourceTracker, v1beta1.ResourceTrackerKindVersionKind), nil
+// genResourceTrackerOwnerReference check the related resourceTracker whether have been created.
+// If not, create it. And return the ownerReference of this resourceTracker.
+func (h *appHandler) genResourceTrackerOwnerReference() *metav1.OwnerReference {
+	return metav1.NewControllerRef(h.resourceTracker, v1beta1.ResourceTrackerKindVersionKind)
 }
 
 func (h *appHandler) generateResourceTrackerName() string {
@@ -529,3 +490,196 @@ func (h *appHandler) removeResourceTracker(ctx context.Context) (bool, error) {
 	h.app.Status.ResourceTracker = nil
 	return true, nil
 }
+
+func (h *appHandler) recodeTrackedWorkload(comp *v1alpha2.Component, compRevisionName string) error {
+	workloadName, err := h.getWorkloadName(comp.Spec.Workload, comp.Name, compRevisionName)
+	if err != nil {
+		return err
+	}
+	if err = h.recodeTrackedResource(workloadName, comp.Spec.Workload); err != nil {
+		return err
+	}
+	return nil
+}
+
+// checkResourceTrackerForTrait check component trait namespace, if it's namespace is different with application, set resourceTracker as its ownerReference
+// and recode trait in handler acrossNamespace field
+func (h *appHandler) checkResourceTrackerForTrait(ctx context.Context, comp v1alpha2.ApplicationConfigurationComponent, compName string) error {
+	for i, ct := range comp.Traits {
+		needTracker, err := h.checkAndSetResourceTracker(&comp.Traits[i].Trait)
+		if err != nil {
+			return err
+		}
+		if needTracker {
+			traitName, err := h.getTraitName(ctx, compName, comp.Traits[i].DeepCopy(), &ct.Trait)
+			if err != nil {
+				return err
+			}
+			if err = h.recodeTrackedResource(traitName, ct.Trait); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// getWorkloadName generate workload name. By default the workload's name will be generated by applicationContext, this func is for application controller
+// get name of crossNamespace workload. The logic of this func is same with the way of appConfig generating workloadName
+func (h *appHandler) getWorkloadName(w runtime.RawExtension, componentName string, revisionName string) (string, error) {
+	workload, err := oamutil.RawExtension2Unstructured(&w)
+	if err != nil {
+		return "", err
+	}
+	var revision int = 0
+	if len(revisionName) != 0 {
+		r, err := utils.ExtractRevision(revisionName)
+		if err != nil {
+			return "", err
+		}
+		revision = r
+	}
+	applicationconfiguration.SetAppWorkloadInstanceName(componentName, workload, revision, strconv.FormatBool(h.inplace))
+	return workload.GetName(), nil
+}
+
+// getTraitName generate trait name. By default the trait name will be generated by applicationContext, this func is for application controller
+// get name of crossNamespace trait. The logic of this func is same with the way of appConfig generating traitName
+func (h *appHandler) getTraitName(ctx context.Context, componentName string, ct *v1alpha2.ComponentTrait, t *runtime.RawExtension) (string, error) {
+	trait, err := oamutil.RawExtension2Unstructured(t)
+	if err != nil {
+		return "", err
+	}
+	traitDef, err := oamutil.FetchTraitDefinition(ctx, h.r, h.r.dm, trait)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			return "", errors.Wrapf(err, "cannot find trait definition %q %q %q", trait.GetAPIVersion(), trait.GetKind(), trait.GetName())
+		}
+		traitDef = oamutil.GetDummyTraitDefinition(trait)
+	}
+	traitType := traitDef.Name
+	if strings.Contains(traitType, ".") {
+		traitType = strings.Split(traitType, ".")[0]
+	}
+	traitName := oamutil.GenTraitName(componentName, ct, traitType)
+	return traitName, nil
+}
+
+// recodeTrackedResource append cross namespace resource to apphandler's acrossNamespaceResources field
+func (h *appHandler) recodeTrackedResource(resourceName string, resource runtime.RawExtension) error {
+	u, err := oamutil.RawExtension2Unstructured(&resource)
+	if err != nil {
+		return err
+	}
+	tr := new(v1beta1.TypedReference)
+	tr.Name = resourceName
+	tr.Namespace = u.GetNamespace()
+	tr.APIVersion = u.GetAPIVersion()
+	tr.Kind = u.GetKind()
+	h.acrossNamespaceResources = append(h.acrossNamespaceResources, *tr)
+	return nil
+}
+
+// Now if workloads or traits are in the same namespace with application, applicationContext will take over gc workloads and traits.
+// Here we cover the case in witch a cross namespace component or one of its cross namespace trait is removed from an application.
+func (h *appHandler) garbageCollection(ctx context.Context) error {
+	rt := new(v1beta1.ResourceTracker)
+	err := h.r.Get(ctx, ctypes.NamespacedName{Name: h.generateResourceTrackerName()}, rt)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			// guarantee app status right
+			h.app.Status.ResourceTracker = nil
+			return nil
+		}
+		return err
+	}
+	applied := map[v1beta1.TypedReference]bool{}
+	if len(h.acrossNamespaceResources) == 0 {
+		h.app.Status.ResourceTracker = nil
+		if err := h.r.Delete(ctx, rt); err != nil {
+			return err
+		}
+		return nil
+	}
+	for _, resource := range h.acrossNamespaceResources {
+		applied[resource] = true
+	}
+	for _, ref := range rt.Status.TrackedResources {
+		if !applied[ref] {
+			resource := new(unstructured.Unstructured)
+			resource.SetAPIVersion(ref.APIVersion)
+			resource.SetKind(ref.Kind)
+			resource.SetNamespace(ref.Namespace)
+			resource.SetName(ref.Name)
+			err := h.r.Delete(ctx, resource)
+			if err != nil {
+				if apierrors.IsNotFound(err) {
+					continue
+				}
+				return err
+			}
+		}
+	}
+	// update resourceTracker status, recode applied across-namespace resources
+	rt.Status.TrackedResources = h.acrossNamespaceResources
+	if err := h.r.Status().Update(ctx, rt); err != nil {
+		return err
+	}
+	h.app.Status.ResourceTracker = &runtimev1alpha1.TypedReference{
+		Name:       rt.Name,
+		Kind:       v1beta1.ResourceTrackerGroupKind,
+		APIVersion: v1beta1.ResourceTrackerKindAPIVersion,
+		UID:        rt.UID}
+	return nil
+}
+
+// handleResourceTracker check the namespace of  all workloads and traits
+// if one resource is across-namespace create resourceTracker and set in appHandler field
+func (h *appHandler) handleResourceTracker(ctx context.Context, components []*v1alpha2.Component, ac *v1alpha2.ApplicationConfiguration) error {
+	resourceTracker := new(v1beta1.ResourceTracker)
+	needTracker := false
+	for _, c := range components {
+		u, err := oamutil.RawExtension2Unstructured(&c.Spec.Workload)
+		if err != nil {
+			return err
+		}
+		if checkResourceDiffWithApp(u, h.app.Namespace) {
+			needTracker = true
+			break
+		}
+	}
+outLoop:
+	for _, acComponent := range ac.Spec.Components {
+		for _, t := range acComponent.Traits {
+			u, err := oamutil.RawExtension2Unstructured(&t.Trait)
+			if err != nil {
+				return err
+			}
+			if checkResourceDiffWithApp(u, h.app.Namespace) {
+				needTracker = true
+				break outLoop
+			}
+		}
+	}
+	if needTracker {
+		// check weather related resourceTracker is existed, if not create it
+		err := h.r.Get(ctx, ctypes.NamespacedName{Name: h.generateResourceTrackerName()}, resourceTracker)
+		if err == nil {
+			h.resourceTracker = resourceTracker
+			return nil
+		}
+		if apierrors.IsNotFound(err) {
+			resourceTracker = &v1beta1.ResourceTracker{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: h.generateResourceTrackerName(),
+				},
+			}
+			if err = h.r.Client.Create(ctx, resourceTracker); err != nil {
+				return err
+			}
+			h.resourceTracker = resourceTracker
+			return nil
+		}
+		return err
+	}
+	return nil
+}

test/e2e-test/app_resourcetracker_test.go

@@ -127,7 +127,7 @@ var _ = Describe("Test application cross namespace resource", func() {
 				return fmt.Errorf("appication status error ")
 			}
 			return nil
-		}, time.Second*600, time.Microsecond*300).Should(BeNil())
+		}, time.Second*300, time.Microsecond*300).Should(BeNil())
 		By("check resource is generated correctly")
 		Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil())
 		var workload appsv1.Deployment
@@ -136,6 +136,10 @@ var _ = Describe("Test application cross namespace resource", func() {
 			if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, appContext); err != nil {
 				return fmt.Errorf("cannot generate AppContext %v", err)
 			}
+			checkRt := new(v1beta1.ResourceTracker)
+			if err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), checkRt); err != nil {
+				return err
+			}
 			component := &v1alpha2.Component{}
 			if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: componentName}, component); err != nil {
 				return fmt.Errorf("cannot generate component %v", err)
@@ -158,6 +162,12 @@ var _ = Describe("Test application cross namespace resource", func() {
 			if len(workload.OwnerReferences) != 1 || workload.OwnerReferences[0].UID != resourceTracker.UID {
 				return fmt.Errorf("wrokload ownerreference error")
 			}
+			if len(checkRt.Status.TrackedResources) != 1 {
+				return fmt.Errorf("resourceTracker status recode trackedResource length missmatch")
+			}
+			if checkRt.Status.TrackedResources[0].Name != workload.Name {
+				return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", checkRt.Status.TrackedResources[0].Name, workload.Name)
+			}
 			return nil
 		}, time.Second*50, time.Microsecond*300).Should(BeNil())
 
@@ -296,6 +306,12 @@ var _ = Describe("Test application cross namespace resource", func() {
 			if len(trait.OwnerReferences) != 1 || trait.OwnerReferences[0].UID != resourceTracker.UID {
 				return fmt.Errorf("trait owner reference missmatch")
 			}
+			if len(resourceTracker.Status.TrackedResources) != 1 {
+				return fmt.Errorf("resourceTracker status recode trackedResource length missmatch")
+			}
+			if resourceTracker.Status.TrackedResources[0].Name != trait.Name {
+				return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, trait.Name)
+			}
 			return nil
 		}, time.Second*60, time.Microsecond*300).Should(BeNil())
 	})
@@ -370,6 +386,12 @@ var _ = Describe("Test application cross namespace resource", func() {
 			if len(trait.OwnerReferences) != 1 || trait.OwnerReferences[0].UID != resourceTracker.UID {
 				return fmt.Errorf("trait owner reference missmatch")
 			}
+			if len(resourceTracker.Status.TrackedResources) != 1 {
+				return fmt.Errorf("resourceTracker status recode trackedResource length missmatch")
+			}
+			if resourceTracker.Status.TrackedResources[0].Name != trait.Name {
+				return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, trait.Name)
+			}
 			return nil
 		}, time.Second*60, time.Microsecond*300).Should(BeNil())
 
@@ -500,6 +522,12 @@ var _ = Describe("Test application cross namespace resource", func() {
 			if app.Status.ResourceTracker == nil || app.Status.ResourceTracker.UID != resourceTracker.UID {
 				return fmt.Errorf("app status resourceTracker error")
 			}
+			if len(resourceTracker.Status.TrackedResources) != 1 {
+				return fmt.Errorf("resourceTracker status recode trackedResource length missmatch")
+			}
+			if resourceTracker.Status.TrackedResources[0].Name != crossDeplpoy.Name {
+				return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, crossDeplpoy.Name)
+			}
 			return nil
 		}, time.Second*60, time.Microsecond*300).Should(BeNil())
 		By("update application by delete cross namespace workload, resource tracker will be deleted, then check app status")
@@ -547,7 +575,7 @@ var _ = Describe("Test application cross namespace resource", func() {
 				return fmt.Errorf("error : cross namespace workload still exist")
 			}
 			if app.Status.ResourceTracker != nil {
-				return fmt.Errorf("errror app status resourceTracker")
+				return fmt.Errorf("error app status resourceTracker")
 			}
 			return nil
 		}, time.Second*60, time.Microsecond*300).Should(BeNil())
@@ -625,12 +653,21 @@ var _ = Describe("Test application cross namespace resource", func() {
 				return fmt.Errorf("error workload number %v", err)
 			}
 			workload = depolys.Items[0]
+			if err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker); err != nil {
+				return err
+			}
 			if len(workload.OwnerReferences) != 1 || workload.OwnerReferences[0].UID != resourceTracker.UID {
 				return fmt.Errorf("wrokload ownerreference error")
 			}
 			if workload.Spec.Template.Spec.Containers[0].Image != "busybox" {
 				return fmt.Errorf("container image not match")
 			}
+			if len(resourceTracker.Status.TrackedResources) != 1 {
+				return fmt.Errorf("resourceTracker status recode trackedResource length missmatch")
+			}
+			if resourceTracker.Status.TrackedResources[0].Name != workload.Name {
+				return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, workload.Name)
+			}
 			return nil
 		}, time.Second*50, time.Microsecond*300).Should(BeNil())
 
@@ -679,6 +716,15 @@ var _ = Describe("Test application cross namespace resource", func() {
 			if workload.Spec.Template.Spec.Containers[0].Image != "nginx" {
 				return fmt.Errorf("container image not match")
 			}
+			if err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker); err != nil {
+				return err
+			}
+			if len(resourceTracker.Status.TrackedResources) != 1 {
+				return fmt.Errorf("resourceTracker status recode trackedResource length missmatch")
+			}
+			if resourceTracker.Status.TrackedResources[0].Name != workload.Name {
+				return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, workload.Name)
+			}
 			return nil
 		}, time.Second*60, time.Microsecond*1000).Should(BeNil())
 
@@ -704,6 +750,445 @@ var _ = Describe("Test application cross namespace resource", func() {
 			return nil
 		}, time.Second*30, time.Microsecond*300).Should(BeNil())
 	})
+
+	It("Test cross-namespace resource gc logic, delete a cross-ns component", func() {
+		var (
+			appName        = "test-app-6"
+			app            = new(v1beta1.Application)
+			component1Name = "test-app-6-comp-1"
+			component2Name = "test-app-6-comp-2"
+		)
+		By("install related definition")
+
+		crossCdJson, err := yaml.YAMLToJSON([]byte(crossCompDefYaml))
+		Expect(err).Should(BeNil())
+		ctd := new(v1beta1.ComponentDefinition)
+		Expect(json.Unmarshal(crossCdJson, ctd)).Should(BeNil())
+		Expect(k8sClient.Create(ctx, ctd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
+
+		app = &v1beta1.Application{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      appName,
+				Namespace: namespace,
+			},
+			Spec: v1beta1.ApplicationSpec{
+				Components: []v1beta1.ApplicationComponent{
+					v1beta1.ApplicationComponent{
+						Name:       component1Name,
+						Type:       "cross-worker",
+						Properties: runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
+					},
+					v1beta1.ApplicationComponent{
+						Name:       component2Name,
+						Type:       "cross-worker",
+						Properties: runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
+					},
+				},
+			},
+		}
+
+		Expect(k8sClient.Create(ctx, app)).Should(BeNil())
+		time.Sleep(3 * time.Second) // give informer cache to sync
+		resourceTracker := new(v1beta1.ResourceTracker)
+
+		By("create application will generate two workload, and generate resourceTracker")
+		Eventually(func() error {
+			app = new(v1beta1.Application)
+			if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app); err != nil {
+				return fmt.Errorf("error to get application %v", err)
+			}
+			if app.Status.Phase != common.ApplicationRunning {
+				return fmt.Errorf("application status not running")
+			}
+			err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker)
+			if err != nil {
+				return fmt.Errorf("error to generate resourceTracker %v", err)
+			}
+			crossOpts := []client.ListOption{
+				client.InNamespace(crossNamespace),
+				client.MatchingLabels{
+					oam.LabelAppName: appName,
+				},
+			}
+			//same, cross := new(appsv1.DeploymentList), new(appsv1.DeploymentList)
+			workloads := new(appsv1.DeploymentList)
+			err = k8sClient.List(ctx, workloads, crossOpts...)
+			if err != nil || len(workloads.Items) != 2 {
+				return fmt.Errorf("failed get workloads")
+			}
+			deploy1 := workloads.Items[0]
+			if len(deploy1.OwnerReferences) != 1 || deploy1.OwnerReferences[0].Kind != v1beta1.ResourceTrackerKind {
+				return fmt.Errorf("deploy1 have error ownerReference")
+			}
+			deploy2 := workloads.Items[1]
+			if len(deploy2.OwnerReferences) != 1 || deploy2.OwnerReferences[0].UID != resourceTracker.UID {
+				return fmt.Errorf("deploy2 have error ownerReference")
+			}
+			if app.Status.ResourceTracker == nil || app.Status.ResourceTracker.UID != resourceTracker.UID {
+				return fmt.Errorf("app status resourceTracker error")
+			}
+			if len(resourceTracker.Status.TrackedResources) != 2 {
+				return fmt.Errorf("resourceTracker status recode trackedResource length missmatch")
+			}
+			if resourceTracker.Status.TrackedResources[0].Namespace != crossNamespace || resourceTracker.Status.TrackedResources[1].Namespace != crossNamespace {
+				return fmt.Errorf("resourceTracker recorde namespace mismatch")
+			}
+			if resourceTracker.Status.TrackedResources[0].Name != deploy1.Name && resourceTracker.Status.TrackedResources[1].Name != deploy1.Name {
+				return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, deploy1.Name)
+			}
+			if resourceTracker.Status.TrackedResources[0].Name != deploy2.Name && resourceTracker.Status.TrackedResources[1].Name != deploy2.Name {
+				return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, deploy2.Name)
+			}
+			return nil
+		}, time.Second*60, time.Microsecond*300).Should(BeNil())
+		By("update application by delete a cross namespace workload, resource tracker will still exist, then check app status")
+		Eventually(func() error {
+			app = new(v1beta1.Application)
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil())
+			app.Spec.Components = app.Spec.Components[:1] // delete a component
+			return k8sClient.Update(ctx, app)
+		}, time.Second*30, time.Microsecond*300).Should(BeNil())
+		Eventually(func() error {
+			app = new(v1beta1.Application)
+			if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app); err != nil {
+				return fmt.Errorf("error to get application %v", err)
+			}
+			if app.Status.Phase != common.ApplicationRunning {
+				return fmt.Errorf("application status not running")
+			}
+			err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker)
+			if err != nil {
+				return fmt.Errorf("failed to get resourceTracker %v", err)
+			}
+			crossOpts := []client.ListOption{
+				client.InNamespace(crossNamespace),
+				client.MatchingLabels{
+					oam.LabelAppName: appName,
+				},
+			}
+			workloads := new(appsv1.DeploymentList)
+			err = k8sClient.List(ctx, workloads, crossOpts...)
+			if err != nil || len(workloads.Items) != 1 {
+				return fmt.Errorf("failed get cross namespace workload")
+			}
+			deploy := workloads.Items[0]
+			if len(deploy.OwnerReferences) != 1 || deploy.OwnerReferences[0].Kind != v1beta1.ResourceTrackerKind {
+				return fmt.Errorf("same ns deploy have error ownerReference")
+			}
+			checkRt := new(v1beta1.ResourceTracker)
+			err = k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), checkRt)
+			if err != nil {
+				return fmt.Errorf("error get resourceTracker")
+			}
+			if app.Status.ResourceTracker == nil {
+				return fmt.Errorf("app status resourceTracker error")
+			}
+			if app.Status.ResourceTracker.UID != checkRt.UID {
+				return fmt.Errorf("error app status resourceTracker UID")
+			}
+			if len(checkRt.Status.TrackedResources) != 1 {
+				return fmt.Errorf("error resourceTracker  status trackedResource")
+			}
+			return nil
+		}, time.Second*80, time.Microsecond*300).Should(BeNil())
+
+		By("deleting application will remove resourceTracker and related resourceTracker will be removed")
+		app = new(v1beta1.Application)
+		Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil())
+		Expect(k8sClient.Delete(ctx, app)).Should(BeNil())
+		Eventually(func() error {
+			err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker)
+			if err == nil {
+				return fmt.Errorf("resourceTracker still exist")
+			}
+			if !apierrors.IsNotFound(err) {
+				return err
+			}
+			return nil
+		}, time.Second*60, time.Microsecond*300).Should(BeNil())
+	})
+
+	It("Test cross-namespace resource gc logic, delete a cross-ns trait", func() {
+		var (
+			appName       = "test-app-7"
+			app           = new(v1beta1.Application)
+			componentName = "test-app-7-comp"
+		)
+		By("install related definition")
+
+		crossCdJson, err := yaml.YAMLToJSON([]byte(crossCompDefYaml))
+		Expect(err).Should(BeNil())
+		ctd := new(v1beta1.ComponentDefinition)
+		Expect(json.Unmarshal(crossCdJson, ctd)).Should(BeNil())
+		Expect(k8sClient.Create(ctx, ctd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
+
+		crossTdJson, err := yaml.YAMLToJSON([]byte(crossNsTdYaml))
+		Expect(err).Should(BeNil())
+		td := new(v1beta1.TraitDefinition)
+		Expect(json.Unmarshal(crossTdJson, td)).Should(BeNil())
+		Expect(k8sClient.Create(ctx, td)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
+
+		app = &v1beta1.Application{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      appName,
+				Namespace: namespace,
+			},
+			Spec: v1beta1.ApplicationSpec{
+				Components: []v1beta1.ApplicationComponent{
+					v1beta1.ApplicationComponent{
+						Name:       componentName,
+						Type:       "cross-worker",
+						Properties: runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
+						Traits: []v1beta1.ApplicationTrait{
+							v1beta1.ApplicationTrait{
+								Type:       "cross-scaler",
+								Properties: runtime.RawExtension{Raw: []byte(`{"replicas": 0}`)},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		Expect(k8sClient.Create(ctx, app)).Should(BeNil())
+		time.Sleep(3 * time.Second) // give informer cache to sync
+		resourceTracker := new(v1beta1.ResourceTracker)
+		By("create app and check resource and app status")
+		Eventually(func() error {
+			app = new(v1beta1.Application)
+			if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app); err != nil {
+				return fmt.Errorf("error to get application %v", err)
+			}
+			if app.Status.Phase != common.ApplicationRunning {
+				return fmt.Errorf("application status not running")
+			}
+			err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker)
+			if err != nil {
+				return fmt.Errorf("error to get resourceTracker %v", err)
+			}
+			mts := new(v1alpha2.ManualScalerTraitList)
+			opts := []client.ListOption{
+				client.InNamespace(crossNamespace),
+				client.MatchingLabels{
+					oam.LabelAppName: appName,
+				},
+			}
+			err = k8sClient.List(ctx, mts, opts...)
+			if err != nil || len(mts.Items) != 1 {
+				return fmt.Errorf("failed generate cross namespace trait")
+			}
+			if len(resourceTracker.Status.TrackedResources) != 2 {
+				return fmt.Errorf("resourceTracker status recode trackedResource length missmatch")
+			}
+			trait := mts.Items[0]
+			if len(trait.OwnerReferences) != 1 || trait.OwnerReferences[0].UID != resourceTracker.UID {
+				return fmt.Errorf("trait owner reference missmatch")
+			}
+			deploys := new(appsv1.DeploymentList)
+			err = k8sClient.List(ctx, deploys, opts...)
+			if err != nil || len(deploys.Items) != 1 {
+				return fmt.Errorf("error to list deploy")
+			}
+			deploy := deploys.Items[0]
+			if deploy.OwnerReferences[0].UID != resourceTracker.UID {
+				return fmt.Errorf("deploy owner reference missmatch")
+			}
+			for _, resource := range resourceTracker.Status.TrackedResources {
+				if resource.Kind == deploy.Kind && resource.Name != deploy.Name {
+					return fmt.Errorf("deploy name mismatch ")
+				}
+				if resource.Kind == trait.Kind && resource.Name != trait.Name {
+					return fmt.Errorf("trait name mismatch")
+				}
+			}
+			return nil
+		}, time.Second*60, time.Microsecond*300).Should(BeNil())
+
+		By("update application trait by delete cross ns trait, resourceTracker will still exist")
+		Eventually(func() error {
+			app = new(v1beta1.Application)
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil())
+			app.Spec.Components[0].Traits = []v1beta1.ApplicationTrait{}
+			return k8sClient.Update(ctx, app)
+		}, time.Second*30, time.Microsecond*300).Should(BeNil())
+
+		Eventually(func() error {
+			app = new(v1beta1.Application)
+			if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app); err != nil {
+				return fmt.Errorf("error to get application %v", err)
+			}
+			if app.Status.Phase != common.ApplicationRunning {
+				return fmt.Errorf("application status not running")
+			}
+			err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker)
+			if err != nil {
+				return fmt.Errorf("error to get resourceTracker %v", err)
+			}
+			mts := new(v1alpha2.ManualScalerTraitList)
+			opts := []client.ListOption{
+				client.InNamespace(crossNamespace),
+				client.MatchingLabels{
+					oam.LabelAppName: appName,
+				},
+			}
+			err = k8sClient.List(ctx, mts, opts...)
+			if err != nil || len(mts.Items) != 0 {
+				return fmt.Errorf("cross namespace trait still exist")
+			}
+			if len(resourceTracker.Status.TrackedResources) != 1 {
+				return fmt.Errorf("resourceTracker status recode trackedResource length missmatch")
+			}
+			deploys := new(appsv1.DeploymentList)
+			err = k8sClient.List(ctx, deploys, opts...)
+			if err != nil || len(deploys.Items) != 1 {
+				return fmt.Errorf("error to list deploy")
+			}
+			deploy := deploys.Items[0]
+			if len(deploy.OwnerReferences) != 1 || deploy.OwnerReferences[0].UID != resourceTracker.UID {
+				return fmt.Errorf("deploy owner reference missmatch")
+			}
+			if resourceTracker.Status.TrackedResources[0].Name != deploy.Name {
+				return fmt.Errorf("error to record deploy name in app status")
+			}
+			return nil
+		}, time.Second*60, time.Microsecond*300).Should(BeNil())
+		By("deleting application will remove resourceTracker and related resourceTracker will be removed")
+		app = new(v1beta1.Application)
+		Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil())
+		Expect(k8sClient.Delete(ctx, app)).Should(BeNil())
+		Eventually(func() error {
+			err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker)
+			if err == nil {
+				return fmt.Errorf("resourceTracker still exist")
+			}
+			if !apierrors.IsNotFound(err) {
+				return err
+			}
+			return nil
+		}, time.Second*60, time.Microsecond*300).Should(BeNil())
+	})
+
+	It("Test cross-namespace resource gc logic, update a cross-ns workload's namespace", func() {
+		// install  related definition
+		crossCdJson, _ := yaml.YAMLToJSON([]byte(crossCompDefYaml))
+		ccd := new(v1beta1.ComponentDefinition)
+		Expect(json.Unmarshal(crossCdJson, ccd)).Should(BeNil())
+		Expect(k8sClient.Create(ctx, ccd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
+
+		normalCdJson, _ := yaml.YAMLToJSON([]byte(normalCompDefYaml))
+		ncd := new(v1beta1.ComponentDefinition)
+		Expect(json.Unmarshal(normalCdJson, ncd)).Should(BeNil())
+		Expect(k8sClient.Create(ctx, ncd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
+
+		var (
+			appName       = "test-app-8"
+			app           = new(v1beta1.Application)
+			componentName = "test-app-8-comp"
+		)
+		app = &v1beta1.Application{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      appName,
+				Namespace: namespace,
+			},
+			Spec: v1beta1.ApplicationSpec{
+				Components: []v1beta1.ApplicationComponent{
+					v1beta1.ApplicationComponent{
+						Name:       componentName,
+						Type:       "cross-worker",
+						Properties: runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
+					},
+				},
+			},
+		}
+		Expect(k8sClient.Create(ctx, app)).Should(BeNil())
+		By("check resource tracker has been created and app status ")
+		resourceTracker := new(v1beta1.ResourceTracker)
+		Eventually(func() error {
+			app := new(v1beta1.Application)
+			if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app); err != nil {
+				return fmt.Errorf("app not found %v", err)
+			}
+			if err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker); err != nil {
+				return err
+			}
+			if app.Status.Phase != common.ApplicationRunning {
+				return fmt.Errorf("application status is not running")
+			}
+			if app.Status.ResourceTracker == nil || app.Status.ResourceTracker.UID != resourceTracker.UID {
+				return fmt.Errorf("appication status error ")
+			}
+			return nil
+		}, time.Second*60, time.Microsecond*300).Should(BeNil())
+		By("check resource is generated correctly")
+		Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil())
+		var workload appsv1.Deployment
+		Eventually(func() error {
+			checkRt := new(v1beta1.ResourceTracker)
+			if err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), checkRt); err != nil {
+				return err
+			}
+			depolys := new(appsv1.DeploymentList)
+			opts := []client.ListOption{
+				client.InNamespace(crossNamespace),
+				client.MatchingLabels{
+					oam.LabelAppName: appName,
+				},
+			}
+			err := k8sClient.List(ctx, depolys, opts...)
+			if err != nil || len(depolys.Items) != 1 {
+				return fmt.Errorf("error workload number %v", err)
+			}
+			workload = depolys.Items[0]
+			if len(workload.OwnerReferences) != 1 || workload.OwnerReferences[0].UID != resourceTracker.UID {
+				return fmt.Errorf("wrokload ownerreference error")
+			}
+			if len(checkRt.Status.TrackedResources) != 1 {
+				return fmt.Errorf("resourceTracker status recode trackedResource length missmatch")
+			}
+			if checkRt.Status.TrackedResources[0].Name != workload.Name {
+				return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", checkRt.Status.TrackedResources[0].Name, workload.Name)
+			}
+			return nil
+		}, time.Second*60, time.Microsecond*300).Should(BeNil())
+
+		By("update application modify workload namespace  will remove resourceTracker and related old workload will be removed")
+		time.Sleep(3 * time.Second) // wait informer cache to be synced
+		Eventually(func() error {
+			app = new(v1beta1.Application)
+			err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)
+			if err != nil {
+				return err
+			}
+			app.Spec.Components[0].Type = "normal-worker"
+			err = k8sClient.Update(ctx, app)
+			if err != nil {
+				return err
+			}
+			return nil
+		}, time.Second*30, time.Microsecond).Should(BeNil())
+		Eventually(func() error {
+			err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker)
+			if err == nil {
+				return fmt.Errorf("resourceTracker still exist")
+			}
+			if !apierrors.IsNotFound(err) {
+				return err
+			}
+			err = k8sClient.Get(ctx, types.NamespacedName{Namespace: crossNamespace, Name: workload.GetName()}, &workload)
+			if err == nil {
+				return fmt.Errorf("wrokload still exist")
+			}
+			if !apierrors.IsNotFound(err) {
+				return err
+			}
+			newWorkload := new(appsv1.Deployment)
+			err = k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: workload.GetName()}, newWorkload)
+			if err != nil {
+				return fmt.Errorf("generate same namespace workload error")
+			}
+			return nil
+		}, time.Second*60, time.Microsecond*300).Should(BeNil())
+	})
 })
 
 func generateResourceTrackerKey(namespace string, name string) types.NamespacedName {