Feat: add compatibility code for new rt (#2920)

Signed-off-by: Yin Da <yd219913@alibaba-inc.com>

apis/core.oam.dev/v1beta1/resourcetracker_types.go

@@ -41,6 +41,10 @@ import (
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // An ResourceTracker represents a tracker for track cross namespace resources
+// +kubebuilder:printcolumn:name="TYPE",type=string,JSONPath=`.spec.type`
+// +kubebuilder:printcolumn:name="APP",type=string,JSONPath=`.metadata.labels['app\.oam\.dev\/name']`
+// +kubebuilder:printcolumn:name="APP-NS",type=string,JSONPath=`.metadata.labels['app\.oam\.dev\/namespace']`
+// +kubebuilder:printcolumn:name="APP-GEN",type=number,JSONPath=`.spec.applicationGeneration`
 // +kubebuilder:resource:scope=Cluster,categories={oam},shortName=rt
 type ResourceTracker struct {
 	metav1.TypeMeta   `json:",inline"`

charts/vela-core/crds/core.oam.dev_resourcetrackers.yaml

@@ -19,7 +19,20 @@ spec:
     singular: resourcetracker
   scope: Cluster
   versions:
-  - name: v1beta1
+  - additionalPrinterColumns:
+    - jsonPath: .spec.type
+      name: TYPE
+      type: string
+    - jsonPath: .metadata.labels['app\.oam\.dev\/name']
+      name: APP
+      type: string
+    - jsonPath: .metadata.labels['app\.oam\.dev\/namespace']
+      name: APP-NS
+      type: string
+    - jsonPath: .spec.applicationGeneration
+      name: APP-GEN
+      type: number
+    name: v1beta1
     schema:
       openAPIV3Schema:
         description: An ResourceTracker represents a tracker for track cross namespace

charts/vela-minimal/crds/core.oam.dev_resourcetrackers.yaml

@@ -19,7 +19,20 @@ spec:
     singular: resourcetracker
   scope: Cluster
   versions:
-  - name: v1beta1
+  - additionalPrinterColumns:
+    - jsonPath: .spec.type
+      name: TYPE
+      type: string
+    - jsonPath: .metadata.labels['app\.oam\.dev\/name']
+      name: APP
+      type: string
+    - jsonPath: .metadata.labels['app\.oam\.dev\/namespace']
+      name: APP-NS
+      type: string
+    - jsonPath: .spec.applicationGeneration
+      name: APP-GEN
+      type: number
+    name: v1beta1
     schema:
       openAPIV3Schema:
         description: An ResourceTracker represents a tracker for track cross namespace

go.mod

@@ -35,6 +35,7 @@ require (
 	github.com/google/uuid v1.1.2
 	github.com/gosuri/uilive v0.0.4
 	github.com/gosuri/uitable v0.0.4
+	github.com/hashicorp/go-version v1.3.0
 	github.com/hashicorp/hcl/v2 v2.9.1
 	github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174
 	github.com/imdario/mergo v0.3.12

go.sum

@@ -921,6 +921,8 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw=
+github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=

legacy/charts/vela-core-legacy/crds/core.oam.dev_resourcetrackers.yaml

@@ -19,7 +19,20 @@ spec:
     singular: resourcetracker
   scope: Cluster
   versions:
-  - name: v1beta1
+  - additionalPrinterColumns:
+    - jsonPath: .spec.type
+      name: TYPE
+      type: string
+    - jsonPath: .metadata.labels['app\.oam\.dev\/name']
+      name: APP
+      type: string
+    - jsonPath: .metadata.labels['app\.oam\.dev\/namespace']
+      name: APP-NS
+      type: string
+    - jsonPath: .spec.applicationGeneration
+      name: APP-GEN
+      type: number
+    name: v1beta1
     schema:
       openAPIV3Schema:
         description: An ResourceTracker represents a tracker for track cross namespace

pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go

@@ -70,12 +70,8 @@ const (
 	// baseWorkflowBackoffWaitTime is the time to wait gc check
 	baseGCBackoffWaitTime = 3000 * time.Millisecond
 
-	legacyResourceTrackerFinalizer = "resourceTracker.finalizer.core.oam.dev"
 	// resourceTrackerFinalizer is to delete the resource tracker of the latest app revision.
 	resourceTrackerFinalizer = "app.oam.dev/resource-tracker-finalizer"
-	// legacyOnlyRevisionFinalizer is to delete all resource trackers of app revisions which may be used
-	// out of the domain of app controller, e.g., AppRollout controller.
-	legacyOnlyRevisionFinalizer = "app.oam.dev/only-revision-finalizer"
 )
 
 // Reconciler reconciles an Application object
@@ -115,7 +111,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	logCtx.AddTag("resource_version", app.ResourceVersion)
 	ctx = oamutil.SetNamespaceInCtx(ctx, app.Namespace)
 	logCtx.SetContext(ctx)
-	metav1.SetMetaDataAnnotation(&app.ObjectMeta, oam.AnnotationKubeVelaVersion, version.VelaVersion)
+	if annotations := app.GetAnnotations(); annotations == nil || annotations[oam.AnnotationKubeVelaVersion] == "" {
+		metav1.SetMetaDataAnnotation(&app.ObjectMeta, oam.AnnotationKubeVelaVersion, version.VelaVersion)
+	}
 	appParser := appfile.NewApplicationParser(r.Client, r.dm, r.pd)
 	handler, err := NewAppHandler(logCtx, r, app, appParser)
 	if err != nil {
@@ -300,7 +298,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 func (r *Reconciler) gcResourceTrackers(logCtx monitorContext.Context, handler *AppHandler, phase common.ApplicationPhase, gcOutdated bool) (ctrl.Result, error) {
 	var options []resourcekeeper.GCOption
 	if !gcOutdated {
-		options = append(options, resourcekeeper.DisableMarkStageGCOption{}, resourcekeeper.DisableGCComponentRevisionOption{})
+		options = append(options, resourcekeeper.DisableMarkStageGCOption{}, resourcekeeper.DisableGCComponentRevisionOption{}, resourcekeeper.DisableLegacyGCOption{})
 	}
 	finished, waiting, err := handler.resourceKeeper.GarbageCollect(logCtx, options...)
 	if err != nil {
@@ -332,31 +330,19 @@ func (r *Reconciler) handleFinalizers(ctx monitorContext.Context, app *v1beta1.A
 			return true, ctrl.Result{}, errors.Wrap(r.Client.Update(ctx, app), errUpdateApplicationFinalizer)
 		}
 	} else {
-		if meta.FinalizerExists(app, legacyResourceTrackerFinalizer) {
-			// TODO(roywang) legacyResourceTrackerFinalizer will be deprecated in the future
-			// this is for backward compatibility
-			rt := &v1beta1.ResourceTracker{}
-			rt.SetName(fmt.Sprintf("%s-%s", app.Namespace, app.Name))
-			if err := r.Client.Delete(ctx, rt); err != nil && !kerrors.IsNotFound(err) {
-				ctx.Error(err, "Failed to delete legacy resource tracker", "name", rt.Name)
-				return true, ctrl.Result{}, errors.WithMessage(err, "cannot remove finalizer")
-			}
-			meta.RemoveFinalizer(app, legacyResourceTrackerFinalizer)
-			return true, ctrl.Result{}, errors.Wrap(r.Client.Update(ctx, app), errUpdateApplicationFinalizer)
-		}
-		if meta.FinalizerExists(app, resourceTrackerFinalizer) || meta.FinalizerExists(app, legacyOnlyRevisionFinalizer) {
+		if meta.FinalizerExists(app, resourceTrackerFinalizer) {
 			rootRT, currentRT, historyRTs, cvRT, err := resourcetracker.ListApplicationResourceTrackers(ctx, r.Client, app)
 			if err != nil {
 				return true, ctrl.Result{}, err
 			}
+			result, err := r.gcResourceTrackers(ctx, handler, common.ApplicationDeleting, true)
+			if err != nil {
+				return true, result, err
+			}
 			if rootRT == nil && currentRT == nil && len(historyRTs) == 0 && cvRT == nil {
 				meta.RemoveFinalizer(app, resourceTrackerFinalizer)
-				// legacyOnlyRevisionFinalizer will be deprecated in the future
-				// this is for backward compatibility
-				meta.RemoveFinalizer(app, legacyOnlyRevisionFinalizer)
 				return true, ctrl.Result{}, errors.Wrap(r.Client.Update(ctx, app), errUpdateApplicationFinalizer)
 			}
-			result, err := r.gcResourceTrackers(ctx, handler, common.ApplicationDeleting, true)
 			return true, result, err
 		}
 	}

pkg/multicluster/fake.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2021 The KubeVela Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package multicluster
+
+import (
+	"context"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// FakeClient set default client and multicluster clients
+type FakeClient struct {
+	client.Client
+	clients map[string]client.Client
+}
+
+// NewFakeClient create a new fake client
+func NewFakeClient(baseClient client.Client) *FakeClient {
+	return &FakeClient{Client: baseClient, clients: map[string]client.Client{}}
+}
+
+// AddCluster add cluster to client map
+func (c *FakeClient) AddCluster(cluster string, cli client.Client) {
+	c.clients[cluster] = cli
+}
+
+func (c *FakeClient) getClient(ctx context.Context) client.Client {
+	cluster := ClusterNameInContext(ctx)
+	if cli, exists := c.clients[cluster]; exists {
+		return cli
+	}
+	return c.Client
+}
+
+// Get retrieves an obj for the given object key from the Kubernetes Cluster.
+// obj must be a struct pointer so that obj can be updated with the response
+// returned by the Server.
+func (c *FakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error {
+	return c.getClient(ctx).Get(ctx, key, obj)
+}
+
+// List retrieves list of objects for a given namespace and list options. On a
+// successful call, Items field in the list will be populated with the
+// result returned from the server.
+func (c *FakeClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
+	return c.getClient(ctx).List(ctx, list, opts...)
+}
+
+// Create saves the object obj in the Kubernetes cluster.
+func (c *FakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
+	return c.getClient(ctx).Create(ctx, obj, opts...)
+}
+
+// Delete deletes the given obj from Kubernetes cluster.
+func (c *FakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
+	return c.getClient(ctx).Delete(ctx, obj, opts...)
+}
+
+// Update updates the given obj in the Kubernetes cluster. obj must be a
+// struct pointer so that obj can be updated with the content returned by the Server.
+func (c *FakeClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
+	return c.getClient(ctx).Update(ctx, obj, opts...)
+}
+
+// Patch patches the given obj in the Kubernetes cluster. obj must be a
+// struct pointer so that obj can be updated with the content returned by the Server.
+func (c *FakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
+	return c.getClient(ctx).Patch(ctx, obj, patch, opts...)
+}
+
+// DeleteAllOf deletes all objects of the given type matching the given options.
+func (c *FakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error {
+	return c.getClient(ctx).DeleteAllOf(ctx, obj, opts...)
+}

pkg/resourcekeeper/gc.go

@@ -18,17 +18,23 @@ package resourcekeeper
 
 import (
 	"context"
+	"encoding/json"
 
 	"github.com/crossplane/crossplane-runtime/pkg/meta"
+	version "github.com/hashicorp/go-version"
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/apps/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha1"
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
 	"github.com/oam-dev/kubevela/pkg/multicluster"
 	"github.com/oam-dev/kubevela/pkg/oam"
 	"github.com/oam-dev/kubevela/pkg/resourcetracker"
+	version2 "github.com/oam-dev/kubevela/version"
 )
 
 // GCOption option for gc
@@ -43,6 +49,7 @@ type gcConfig struct {
 	disableSweep               bool
 	disableFinalize            bool
 	disableComponentRevisionGC bool
+	disableLegacyGC            bool
 }
 
 func newGCConfig(options ...GCOption) *gcConfig {
@@ -111,6 +118,12 @@ func (h *resourceKeeper) garbageCollect(ctx context.Context, cfg *gcConfig) (fin
 			return false, waiting, errors.Wrapf(err, "failed to garbage collect component revisions in unused components")
 		}
 	}
+	// Garbage Collect Legacy ResourceTrackers
+	if !cfg.disableLegacyGC {
+		if err = gc.GarbageCollectLegacyResourceTrackers(ctx); err != nil {
+			return false, waiting, errors.Wrapf(err, "failed to garbage collect legacy resource trackers")
+		}
+	}
 	return finished, waiting, nil
 }
 
@@ -273,3 +286,68 @@ func (h *gcHandler) GarbageCollectComponentRevisionResourceTracker(ctx context.C
 	}
 	return nil
 }
+
+const velaVersionNumberToUpgradeResourceTracker = "v1.2.0"
+
+func (h *gcHandler) GarbageCollectLegacyResourceTrackers(ctx context.Context) error {
+	// skip legacy gc if application is not handled by new version rt
+	if h.app.GetDeletionTimestamp() == nil && h.resourceKeeper._currentRT == nil {
+		return nil
+	}
+	// check app version
+	velaVersionToUpgradeResourceTracker, _ := version.NewVersion(velaVersionNumberToUpgradeResourceTracker)
+	var currentVersionNumber string
+	if annotations := h.app.GetAnnotations(); annotations != nil && annotations[oam.AnnotationKubeVelaVersion] != "" {
+		currentVersionNumber = annotations[oam.AnnotationKubeVelaVersion]
+	}
+	if currentVersionNumber == "UNKNOWN" {
+		return nil
+	}
+	currentVersion, err := version.NewVersion(currentVersionNumber)
+	if err == nil && velaVersionToUpgradeResourceTracker.LessThanOrEqual(currentVersion) {
+		return nil
+	}
+	// remove legacy ResourceTrackers
+	clusters := map[string]bool{multicluster.ClusterLocalName: true}
+	for _, rsc := range h.app.Status.AppliedResources {
+		if rsc.Cluster != "" {
+			clusters[rsc.Cluster] = true
+		}
+	}
+	for _, policy := range h.app.Spec.Policies {
+		if policy.Type == v1alpha1.EnvBindingPolicyType {
+			spec := &v1alpha1.EnvBindingSpec{}
+			if err = json.Unmarshal(policy.Properties.Raw, &spec); err == nil {
+				for _, env := range spec.Envs {
+					if env.Placement.ClusterSelector != nil && env.Placement.ClusterSelector.Name != "" {
+						clusters[env.Placement.ClusterSelector.Name] = true
+					}
+				}
+			}
+		}
+	}
+	for cluster := range clusters {
+		_ctx := multicluster.ContextWithClusterName(ctx, cluster)
+		rts := &unstructured.UnstructuredList{}
+		rts.SetGroupVersionKind(v1beta1.SchemeGroupVersion.WithKind("ResourceTracker"))
+		if err = h.Client.List(_ctx, rts, client.MatchingLabels(map[string]string{
+			oam.LabelAppName:      h.app.Name,
+			oam.LabelAppNamespace: h.app.Namespace,
+		})); err != nil {
+			return errors.Wrapf(err, "failed to list resource trackers for app %s/%s in cluster %s", h.app.Namespace, h.app.Name, cluster)
+		}
+		for _, rt := range rts.Items {
+			if s, exists, _ := unstructured.NestedString(rt.Object, "spec", "type"); !exists || s == "" {
+				if err = h.Client.Delete(_ctx, rt.DeepCopy()); err != nil {
+					return errors.Wrapf(err, "failed to delete legacy resource tracker %s for app %s/%s in cluster %s", rt.GetName(), h.app.Namespace, h.app.Name, cluster)
+				}
+			}
+		}
+	}
+	// upgrade app version
+	v12.SetMetaDataAnnotation(&h.app.ObjectMeta, oam.AnnotationKubeVelaVersion, version2.VelaVersion)
+	if err = h.Client.Update(ctx, h.app); err != nil {
+		return errors.Wrapf(err, "failed to upgrade app %s/%s", h.app.Namespace, h.app.Name)
+	}
+	return nil
+}

pkg/resourcekeeper/gc_suite_test.go

@@ -0,0 +1,141 @@
+/*
+Copyright 2021 The KubeVela Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resourcekeeper
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/crossplane/crossplane-runtime/pkg/meta"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
+	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha1"
+	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
+	"github.com/oam-dev/kubevela/pkg/multicluster"
+	"github.com/oam-dev/kubevela/pkg/oam"
+	"github.com/oam-dev/kubevela/pkg/utils"
+	"github.com/oam-dev/kubevela/version"
+)
+
+var _ = Describe("Test ResourceKeeper garbage collection", func() {
+
+	var namespace string
+
+	BeforeEach(func() {
+		namespace = "test-ns-" + utils.RandomString(4)
+		Expect(testClient.Create(context.Background(), &v1.Namespace{ObjectMeta: v12.ObjectMeta{Name: namespace}})).Should(Succeed())
+	})
+
+	AfterEach(func() {
+		ns := &v1.Namespace{}
+		Expect(testClient.Get(context.Background(), types.NamespacedName{Name: namespace}, ns)).Should(Succeed())
+		Expect(testClient.Delete(context.Background(), ns)).Should(Succeed())
+	})
+
+	It("Test gcHandler garbage collect legacy RT", func() {
+		if version.VelaVersion == "UNKNOWN" {
+			version.VelaVersion = velaVersionNumberToUpgradeResourceTracker
+		}
+		ctx := context.Background()
+		cli := multicluster.NewFakeClient(testClient)
+		cli.AddCluster("worker", workerClient)
+		cli.AddCluster("worker-2", workerClient)
+		app := &v1beta1.Application{ObjectMeta: v12.ObjectMeta{Name: "gc-app", Namespace: namespace}}
+		bs, err := json.Marshal(&v1alpha1.EnvBindingSpec{
+			Envs: []v1alpha1.EnvConfig{{
+				Placement: v1alpha1.EnvPlacement{ClusterSelector: &common.ClusterSelector{Name: "worker"}},
+			}},
+		})
+		Expect(err).Should(Succeed())
+		meta.AddAnnotations(app, map[string]string{oam.AnnotationKubeVelaVersion: "v1.1.13"})
+		app.Spec = v1beta1.ApplicationSpec{
+			Components: []common.ApplicationComponent{},
+			Policies: []v1beta1.AppPolicy{{
+				Type:       v1alpha1.EnvBindingPolicyType,
+				Properties: &runtime.RawExtension{Raw: bs},
+			}},
+		}
+		app.Status.AppliedResources = []common.ClusterObjectReference{{
+			Cluster: "worker-2",
+		}}
+		Expect(cli.Create(ctx, app)).Should(Succeed())
+		keeper := &resourceKeeper{Client: cli, app: app}
+		h := gcHandler{resourceKeeper: keeper}
+		rt := &v1beta1.ResourceTracker{}
+		rt.SetName("gc-app-rt-v1-" + namespace)
+		rt.SetLabels(map[string]string{
+			oam.LabelAppName:      h.app.Name,
+			oam.LabelAppNamespace: h.app.Namespace,
+		})
+		rt3 := rt.DeepCopy()
+		rt4 := rt.DeepCopy()
+		rt5 := rt.DeepCopy()
+		rt4.SetName("gc-app-rt-v2-" + namespace)
+		Expect(cli.Create(ctx, rt)).Should(Succeed())
+		rt2 := &v1beta1.ResourceTracker{}
+		rt2.Spec.Type = v1beta1.ResourceTrackerTypeVersioned
+		rt2.SetName("gc-app-rt-v2-" + namespace)
+		rt2.SetLabels(map[string]string{
+			oam.LabelAppName:      h.app.Name,
+			oam.LabelAppNamespace: h.app.Namespace,
+		})
+		Expect(cli.Create(ctx, rt2)).Should(Succeed())
+		Expect(h.GarbageCollectLegacyResourceTrackers(ctx)).Should(Succeed())
+		Expect(cli.Create(multicluster.ContextWithClusterName(ctx, "worker"), rt3)).Should(Succeed())
+		Expect(cli.Create(multicluster.ContextWithClusterName(ctx, "worker-2"), rt4)).Should(Succeed())
+
+		checkRTExists := func(_ctx context.Context, name string, exists bool) {
+			_rt := &v1beta1.ResourceTracker{}
+			err := cli.Get(_ctx, types.NamespacedName{Name: name}, _rt)
+			if exists {
+				Expect(err).Should(Succeed())
+			} else {
+				Expect(errors.IsNotFound(err)).Should(BeTrue())
+			}
+		}
+
+		Expect(h.GarbageCollectLegacyResourceTrackers(ctx)).Should(Succeed())
+		checkRTExists(ctx, rt.GetName(), true)
+		checkRTExists(ctx, rt2.GetName(), true)
+		checkRTExists(multicluster.ContextWithClusterName(ctx, "worker"), rt3.GetName(), true)
+		checkRTExists(multicluster.ContextWithClusterName(ctx, "worker-2"), rt4.GetName(), true)
+
+		h.resourceKeeper._currentRT = rt2
+		Expect(h.GarbageCollectLegacyResourceTrackers(ctx)).Should(Succeed())
+		checkRTExists(ctx, rt.GetName(), false)
+		checkRTExists(ctx, rt2.GetName(), true)
+		checkRTExists(multicluster.ContextWithClusterName(ctx, "worker"), rt3.GetName(), false)
+		checkRTExists(multicluster.ContextWithClusterName(ctx, "worker-2"), rt4.GetName(), false)
+
+		Expect(app.GetAnnotations()[oam.AnnotationKubeVelaVersion]).Should(Equal("v1.2.0"))
+		Expect(cli.Create(ctx, rt5)).Should(Succeed())
+		Expect(h.GarbageCollectLegacyResourceTrackers(ctx)).Should(Succeed())
+		checkRTExists(ctx, rt5.GetName(), true)
+
+		meta.AddAnnotations(app, map[string]string{oam.AnnotationKubeVelaVersion: "UNKNOWN"})
+		Expect(h.GarbageCollectLegacyResourceTrackers(ctx)).Should(Succeed())
+		checkRTExists(ctx, rt5.GetName(), true)
+	})
+
+})

pkg/resourcekeeper/gc_test.go

@@ -146,9 +146,10 @@ func TestResourceKeeperGarbageCollect(t *testing.T) {
 	addConfigMapToRT(4, 3, 3)
 	checkCount(4, 4, 3)
 
+	opts := []GCOption{DisableLegacyGCOption{}}
 	// no need to gc
 	rk := createRK(3, true)
-	finished, _, err := rk.GarbageCollect(ctx)
+	finished, _, err := rk.GarbageCollect(ctx, opts...)
 	r.NoError(err)
 	r.True(finished)
 	checkCount(4, 4, 3)
@@ -158,11 +159,11 @@ func TestResourceKeeperGarbageCollect(t *testing.T) {
 	rtMaps[2].SetDeletionTimestamp(&dt)
 	r.NoError(cli.Update(ctx, rtMaps[2]))
 	rk = createRK(3, true)
-	finished, _, err = rk.GarbageCollect(ctx)
+	finished, _, err = rk.GarbageCollect(ctx, opts...)
 	r.NoError(err)
 	r.False(finished)
 	rk = createRK(3, true)
-	finished, _, err = rk.GarbageCollect(ctx)
+	finished, _, err = rk.GarbageCollect(ctx, opts...)
 	r.NoError(err)
 	r.True(finished)
 	checkCount(3, 3, 3)
@@ -170,18 +171,18 @@ func TestResourceKeeperGarbageCollect(t *testing.T) {
 	// delete cm4, trigger gc for rt3, comp-3 no use
 	r.NoError(cli.Delete(ctx, cmMaps[4]))
 	rk = createRK(4, true)
-	finished, _, err = rk.GarbageCollect(ctx)
+	finished, _, err = rk.GarbageCollect(ctx, opts...)
 	r.NoError(err)
 	r.True(finished)
 	checkCount(2, 2, 2)
 
 	// upgrade and gc legacy rt1
 	rk = createRK(4, false)
-	finished, _, err = rk.GarbageCollect(ctx)
+	finished, _, err = rk.GarbageCollect(ctx, opts...)
 	r.NoError(err)
 	r.False(finished)
 	rk = createRK(4, false)
-	finished, _, err = rk.GarbageCollect(ctx)
+	finished, _, err = rk.GarbageCollect(ctx, opts...)
 	r.NoError(err)
 	r.True(finished)
 	checkCount(0, 1, 0)
@@ -197,17 +198,17 @@ func TestResourceKeeperGarbageCollect(t *testing.T) {
 	checkCount(3, 3, 1)
 	rk = createRK(6, false)
 	rk.app.SetDeletionTimestamp(&dt)
-	finished, _, err = rk.GarbageCollect(ctx)
+	finished, _, err = rk.GarbageCollect(ctx, opts...)
 	r.NoError(err)
 	r.False(finished)
 	rk = createRK(6, false)
-	finished, _, err = rk.GarbageCollect(ctx)
+	finished, _, err = rk.GarbageCollect(ctx, opts...)
 	r.NoError(err)
 	r.True(finished)
 	checkCount(0, 0, 0)
 
 	rk = createRK(7, false)
-	finished, _, err = rk.GarbageCollect(ctx)
+	finished, _, err = rk.GarbageCollect(ctx, opts...)
 	r.NoError(err)
 	r.True(finished)
 }

pkg/resourcekeeper/options.go

@@ -75,6 +75,14 @@ func (option DisableGCComponentRevisionOption) ApplyToGCConfig(cfg *gcConfig) {
 	cfg.disableComponentRevisionGC = true
 }
 
+// DisableLegacyGCOption disable garbage collect legacy resourcetrackers
+type DisableLegacyGCOption struct{}
+
+// ApplyToGCConfig apply change to gc config
+func (option DisableLegacyGCOption) ApplyToGCConfig(cfg *gcConfig) {
+	cfg.disableLegacyGC = true
+}
+
 // GarbageCollectStrategyOption apply garbage collect strategy to resourcetracker recording
 type GarbageCollectStrategyOption v1alpha1.GarbageCollectStrategy
 

pkg/resourcekeeper/suite_test.go

@@ -35,6 +35,9 @@ import (
 var testEnv *envtest.Environment
 var testClient client.Client
 
+var workerEnv *envtest.Environment
+var workerClient client.Client
+
 func TestResourceKeeper(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "ResourceKeeper Suite")
@@ -62,6 +65,22 @@ var _ = BeforeSuite(func(done Done) {
 	Expect(err).ShouldNot(HaveOccurred())
 	Expect(testClient).ShouldNot(BeNil())
 
+	workerEnv = &envtest.Environment{
+		ControlPlaneStartTimeout: time.Minute,
+		ControlPlaneStopTimeout:  time.Minute,
+		CRDDirectoryPaths: []string{
+			filepath.Join("../..", "charts/vela-core/crds"), // this has all the required CRDs,
+		},
+		UseExistingCluster:    pointer.Bool(false),
+		ErrorIfCRDPathMissing: true,
+	}
+	cfg, err = workerEnv.Start()
+	Expect(err).ShouldNot(HaveOccurred())
+	Expect(cfg).ShouldNot(BeNil())
+	workerClient, err = client.New(cfg, client.Options{Scheme: common.Scheme})
+	Expect(err).ShouldNot(HaveOccurred())
+	Expect(workerClient).ShouldNot(BeNil())
+
 	close(done)
 }, 300)
 

test/e2e-multicluster-test/multicluster_test.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
+	"github.com/oam-dev/kubevela/pkg/utils"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -53,7 +54,7 @@ func initializeContext() (hubCtx context.Context, workerCtx context.Context) {
 func initializeContextAndNamespace() (hubCtx context.Context, workerCtx context.Context, namespace string) {
 	hubCtx, workerCtx = initializeContext()
 	// initialize test namespace
-	namespace = fmt.Sprintf("test-%d", time.Now().UnixNano())
+	namespace = "test-mc-" + utils.RandomString(4)
 	ns := &v1.Namespace{ObjectMeta: v12.ObjectMeta{Name: namespace}}
 	Expect(k8sClient.Create(hubCtx, ns.DeepCopy())).Should(Succeed())
 	Expect(k8sClient.Create(workerCtx, ns.DeepCopy())).Should(Succeed())

test/e2e-test/resource_policy_test.go

@@ -78,9 +78,11 @@ var _ = Describe("Application Resource-Related Policy Tests", func() {
 		}, 30*time.Second).Should(Succeed())
 
 		By("test apply-once policy")
-		Expect(k8sClient.Get(ctx, appKey, app)).Should(Succeed())
-		app.Spec.Policies[0].Properties = &runtime.RawExtension{Raw: []byte(`{"enable":true}`)}
-		Expect(k8sClient.Update(ctx, app)).Should(Succeed())
+		Eventually(func(g Gomega) {
+			g.Expect(k8sClient.Get(ctx, appKey, app)).Should(Succeed())
+			app.Spec.Policies[0].Properties = &runtime.RawExtension{Raw: []byte(`{"enable":true}`)}
+			g.Expect(k8sClient.Update(ctx, app)).Should(Succeed())
+		}, 10*time.Second).Should(Succeed())
 		Eventually(func(g Gomega) {
 			g.Expect(k8sClient.Get(ctx, appKey, app)).Should(Succeed())
 			g.Expect(app.Status.ObservedGeneration).Should(Equal(app.Generation))