From f9ecc02417c8f9716f06b389ab9ecdffaa897d0a Mon Sep 17 00:00:00 2001 From: Somefive Date: Mon, 29 Nov 2021 16:14:30 +0800 Subject: [PATCH] Feat: rework rt (#2797) --- apis/core.oam.dev/common/types.go | 5 + apis/core.oam.dev/v1beta1/core_types.go | 30 ---- apis/core.oam.dev/v1beta1/resourcetracker.go | 146 ++++++++++++++++++ .../v1beta1/zz_generated.deepcopy.go | 3 +- apis/interfaces/resourcetracker.go | 35 +++++ .../crds/core.oam.dev_resourcetrackers.yaml | 32 +--- .../crds/core.oam.dev_resourcetrackers.yaml | 32 +--- .../crds/core.oam.dev_resourcetrackers.yaml | 32 +--- .../application/application_controller.go | 20 +-- .../v1alpha2/application/dispatch/dispatch.go | 95 ++---------- .../application/dispatch/dispatch_test.go | 64 +++----- .../v1alpha2/application/dispatch/gc_test.go | 133 +++++++++------- .../v1alpha2/application/revision.go | 26 +--- .../v1alpha1/rollout/handler.go | 10 +- pkg/multicluster/gc.go | 83 ++++++---- pkg/multicluster/gc_test.go | 9 +- pkg/policy/envbinding/placement.go | 24 +++ pkg/resourcetracker/resourcetracker.go | 62 ++++++++ pkg/utils/apply/apply.go | 34 ++++ .../providers/multicluster/multicluster.go | 3 + 20 files changed, 511 insertions(+), 367 deletions(-) create mode 100644 apis/core.oam.dev/v1beta1/resourcetracker.go create mode 100644 apis/interfaces/resourcetracker.go create mode 100644 pkg/resourcetracker/resourcetracker.go diff --git a/apis/core.oam.dev/common/types.go b/apis/core.oam.dev/common/types.go index e6bbf6868..d9a2402cc 100644 --- a/apis/core.oam.dev/common/types.go +++ b/apis/core.oam.dev/common/types.go @@ -479,6 +479,11 @@ type ClusterObjectReference struct { corev1.ObjectReference `json:",inline"` } +// Equal check if two references are equal +func (in ClusterObjectReference) Equal(r ClusterObjectReference) bool { + return in.APIVersion == r.APIVersion && in.Kind == r.Kind && in.Name == r.Name && in.Namespace == r.Namespace && in.UID == r.UID && in.Creator == r.Creator && in.Cluster == r.Cluster +} + // RawExtensionPointer is the pointer of raw extension type RawExtensionPointer struct { RawExtension *runtime.RawExtension diff --git a/apis/core.oam.dev/v1beta1/core_types.go b/apis/core.oam.dev/v1beta1/core_types.go index 3a7f819f5..c49ff6578 100644 --- a/apis/core.oam.dev/v1beta1/core_types.go +++ b/apis/core.oam.dev/v1beta1/core_types.go @@ -17,7 +17,6 @@ package v1beta1 import ( - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -257,32 +256,3 @@ type ScopeDefinitionList struct { metav1.ListMeta `json:"metadata,omitempty"` Items []ScopeDefinition `json:"items"` } - -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +genclient -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// An ResourceTracker represents a tracker for track cross namespace resources -// +kubebuilder:resource:scope=Cluster,categories={oam},shortName=tracker -type ResourceTracker struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Status ResourceTrackerStatus `json:"status,omitempty"` -} - -// ResourceTrackerStatus define the status of resourceTracker -type ResourceTrackerStatus struct { - TrackedResources []corev1.ObjectReference `json:"trackedResources,omitempty"` -} - -// +kubebuilder:object:root=true -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// ResourceTrackerList contains a list of ResourceTracker -type ResourceTrackerList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []ResourceTracker `json:"items"` -} diff --git a/apis/core.oam.dev/v1beta1/resourcetracker.go b/apis/core.oam.dev/v1beta1/resourcetracker.go new file mode 100644 index 000000000..a63f2af9a --- /dev/null +++ b/apis/core.oam.dev/v1beta1/resourcetracker.go @@ -0,0 +1,146 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/common" + "github.com/oam-dev/kubevela/apis/interfaces" + "github.com/oam-dev/kubevela/pkg/oam" +) + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// An ResourceTracker represents a tracker for track cross namespace resources +// +kubebuilder:resource:scope=Cluster,categories={oam},shortName=tracker +type ResourceTracker struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Status ResourceTrackerStatus `json:"status,omitempty"` +} + +// ResourceTrackerStatus define the status of resourceTracker +type ResourceTrackerStatus struct { + TrackedResources []common.ClusterObjectReference `json:"trackedResources,omitempty"` +} + +// +kubebuilder:object:root=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ResourceTrackerList contains a list of ResourceTracker +type ResourceTrackerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ResourceTracker `json:"items"` +} + +// ToOwnerReference convert ResourceTracker into owner reference for other resource to refer +func (in *ResourceTracker) ToOwnerReference() *metav1.OwnerReference { + return &metav1.OwnerReference{ + APIVersion: SchemeGroupVersion.String(), + Kind: ResourceTrackerKind, + Name: in.Name, + UID: in.UID, + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + } +} + +// AddOwnerReferenceToTrackerResource add resourcetracker as owner reference to target object, return true if already exists (outdated) +func (in *ResourceTracker) AddOwnerReferenceToTrackerResource(rsc interfaces.ObjectOwner) bool { + ownerRefs := []metav1.OwnerReference{*in.ToOwnerReference()} + exists := false + for _, owner := range rsc.GetOwnerReferences() { + // delete the old resourceTracker owner + if owner.Kind == ResourceTrackerKind && owner.APIVersion == SchemeGroupVersion.String() { + exists = true + continue + } + if owner.Controller != nil && *owner.Controller && owner.UID != in.UID { + owner.Controller = pointer.BoolPtr(false) + } + ownerRefs = append(ownerRefs, owner) + } + rsc.SetOwnerReferences(ownerRefs) + return exists +} + +func (in *ResourceTracker) addClusterObjectReference(ref common.ClusterObjectReference) bool { + for _, _rsc := range in.Status.TrackedResources { + if _rsc.Equal(ref) { + return true + } + } + in.Status.TrackedResources = append(in.Status.TrackedResources, ref) + return false +} + +// AddTrackedResource add new object reference into tracked resources, return if already exists +func (in *ResourceTracker) AddTrackedResource(rsc interfaces.TrackableResource) bool { + return in.addClusterObjectReference(common.ClusterObjectReference{ + ObjectReference: v1.ObjectReference{ + APIVersion: rsc.GetAPIVersion(), + Kind: rsc.GetKind(), + Name: rsc.GetName(), + Namespace: rsc.GetNamespace(), + UID: rsc.GetUID(), + }, + }) +} + +// AddTrackedCluster add resourcetracker in remote cluster into tracked resources, return if already exists +func (in *ResourceTracker) AddTrackedCluster(clusterName string) bool { + if clusterName == "" { + return true + } + return in.addClusterObjectReference(common.ClusterObjectReference{ + Cluster: clusterName, + ObjectReference: v1.ObjectReference{ + APIVersion: SchemeGroupVersion.String(), + Kind: ResourceTrackerKind, + Name: in.GetName(), + }, + }) +} + +// GetTrackedClusters return remote clusters recorded in the resource tracker +func (in *ResourceTracker) GetTrackedClusters() (clusters []string) { + for _, ref := range in.Status.TrackedResources { + if ref.APIVersion == SchemeGroupVersion.String() && ref.Kind == ResourceTrackerKind && ref.Name == in.Name && ref.Cluster != "" { + clusters = append(clusters, ref.Cluster) + } + } + return +} + +// IsLifeLong check if resourcetracker shares the same whole life with the entire application +func (in *ResourceTracker) IsLifeLong() bool { + _, ok := in.GetAnnotations()[oam.AnnotationResourceTrackerLifeLong] + return ok +} + +// SetLifeLong set life long to resource tracker +func (in *ResourceTracker) SetLifeLong() { + in.SetAnnotations(map[string]string{oam.AnnotationResourceTrackerLifeLong: "true"}) +} diff --git a/apis/core.oam.dev/v1beta1/zz_generated.deepcopy.go b/apis/core.oam.dev/v1beta1/zz_generated.deepcopy.go index b2a1423ab..b0e463a30 100644 --- a/apis/core.oam.dev/v1beta1/zz_generated.deepcopy.go +++ b/apis/core.oam.dev/v1beta1/zz_generated.deepcopy.go @@ -21,7 +21,6 @@ limitations under the License. package v1beta1 import ( - corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -1040,7 +1039,7 @@ func (in *ResourceTrackerStatus) DeepCopyInto(out *ResourceTrackerStatus) { *out = *in if in.TrackedResources != nil { in, out := &in.TrackedResources, &out.TrackedResources - *out = make([]corev1.ObjectReference, len(*in)) + *out = make([]common.ClusterObjectReference, len(*in)) copy(*out, *in) } } diff --git a/apis/interfaces/resourcetracker.go b/apis/interfaces/resourcetracker.go new file mode 100644 index 000000000..9cf29a456 --- /dev/null +++ b/apis/interfaces/resourcetracker.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package interfaces + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ObjectOwner is the interface for get and set ownerReference +type ObjectOwner interface { + GetOwnerReferences() []metav1.OwnerReference + SetOwnerReferences([]metav1.OwnerReference) +} + +// TrackableResource is the interface for resources to be tracked by resourcetracker +type TrackableResource interface { + client.Object + metav1.Type + ObjectOwner +} diff --git a/charts/vela-core/crds/core.oam.dev_resourcetrackers.yaml b/charts/vela-core/crds/core.oam.dev_resourcetrackers.yaml index bbf68949b..6b0f2616d 100644 --- a/charts/vela-core/crds/core.oam.dev_resourcetrackers.yaml +++ b/charts/vela-core/crds/core.oam.dev_resourcetrackers.yaml @@ -42,35 +42,17 @@ spec: properties: trackedResources: items: - description: 'ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. - Invalid usage help. It is impossible to add specific help for - individual usage. In most embedded usages, there are particular restrictions - like, "must refer only to types A and B" or "UID not honored" - or "name must be restricted". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual - struct is irrelevant. 5. We cannot easily change it. Because - this type is embedded in many locations, updates to this type will - affect numerous schemas. Don''t make new APIs embed an underspecified - API type they do not control. Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - .' + description: ClusterObjectReference defines the object reference + with cluster. properties: apiVersion: description: API version of the referent. type: string + cluster: + type: string + creator: + description: ResourceCreatorRole defines the resource creator. + type: string fieldPath: description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go diff --git a/charts/vela-minimal/crds/core.oam.dev_resourcetrackers.yaml b/charts/vela-minimal/crds/core.oam.dev_resourcetrackers.yaml index bbf68949b..6b0f2616d 100644 --- a/charts/vela-minimal/crds/core.oam.dev_resourcetrackers.yaml +++ b/charts/vela-minimal/crds/core.oam.dev_resourcetrackers.yaml @@ -42,35 +42,17 @@ spec: properties: trackedResources: items: - description: 'ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. - Invalid usage help. It is impossible to add specific help for - individual usage. In most embedded usages, there are particular restrictions - like, "must refer only to types A and B" or "UID not honored" - or "name must be restricted". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual - struct is irrelevant. 5. We cannot easily change it. Because - this type is embedded in many locations, updates to this type will - affect numerous schemas. Don''t make new APIs embed an underspecified - API type they do not control. Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - .' + description: ClusterObjectReference defines the object reference + with cluster. properties: apiVersion: description: API version of the referent. type: string + cluster: + type: string + creator: + description: ResourceCreatorRole defines the resource creator. + type: string fieldPath: description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go diff --git a/legacy/charts/vela-core-legacy/crds/core.oam.dev_resourcetrackers.yaml b/legacy/charts/vela-core-legacy/crds/core.oam.dev_resourcetrackers.yaml index bbf68949b..6b0f2616d 100644 --- a/legacy/charts/vela-core-legacy/crds/core.oam.dev_resourcetrackers.yaml +++ b/legacy/charts/vela-core-legacy/crds/core.oam.dev_resourcetrackers.yaml @@ -42,35 +42,17 @@ spec: properties: trackedResources: items: - description: 'ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. - Invalid usage help. It is impossible to add specific help for - individual usage. In most embedded usages, there are particular restrictions - like, "must refer only to types A and B" or "UID not honored" - or "name must be restricted". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual - struct is irrelevant. 5. We cannot easily change it. Because - this type is embedded in many locations, updates to this type will - affect numerous schemas. Don''t make new APIs embed an underspecified - API type they do not control. Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - .' + description: ClusterObjectReference defines the object reference + with cluster. properties: apiVersion: description: API version of the referent. type: string + cluster: + type: string + creator: + description: ResourceCreatorRole defines the resource creator. + type: string fieldPath: description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go index 620df47e8..df2cc2ea2 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go @@ -219,7 +219,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if wfStatus != nil { ref, err := handler.DispatchAndGC(ctx) if err == nil { - err = multicluster.GarbageCollectionForOutdatedResourcesInSubClusters(ctx, app, func(c context.Context) error { + err = multicluster.GarbageCollectionForOutdatedResourcesInSubClusters(ctx, r.Client, app, func(c context.Context) error { _, e := handler.DispatchAndGC(c) return e }) @@ -332,23 +332,7 @@ func (r *Reconciler) handleFinalizers(ctx monitorContext.Context, app *v1beta1.A return true, errors.Wrap(r.Client.Update(ctx, app), errUpdateApplicationFinalizer) } if meta.FinalizerExists(app, resourceTrackerFinalizer) || meta.FinalizerExists(app, legacyOnlyRevisionFinalizer) { - listOpts := []client.ListOption{ - client.MatchingLabels{ - oam.LabelAppName: app.Name, - oam.LabelAppNamespace: app.Namespace, - }} - rtList := &v1beta1.ResourceTrackerList{} - if err := r.Client.List(ctx, rtList, listOpts...); err != nil { - ctx.Error(err, "Failed to list resource tracker of app", "name", app.Name) - return true, errors.WithMessage(err, "cannot remove finalizer") - } - for _, rt := range rtList.Items { - if err := r.Client.Delete(ctx, rt.DeepCopy()); err != nil && !kerrors.IsNotFound(err) { - ctx.Error(err, "Failed to delete resource tracker", "name", rt.Name) - return true, errors.WithMessage(err, "cannot remove finalizer") - } - } - if err := multicluster.GarbageCollectionForAllResourceTrackersInSubCluster(ctx, r.Client, app); err != nil { + if err := multicluster.GarbageCollectionForAllResourceTrackers(ctx, r.Client, app); err != nil { return true, err } meta.RemoveFinalizer(app, resourceTrackerFinalizer) diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/dispatch.go b/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/dispatch.go index 49a458608..e1245a380 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/dispatch.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/dispatch.go @@ -23,15 +23,12 @@ import ( "github.com/pkg/errors" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" - "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" + "github.com/oam-dev/kubevela/apis/core.oam.dev/common" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/pkg/oam" "github.com/oam-dev/kubevela/pkg/utils/apply" @@ -205,7 +202,7 @@ func (a *AppManifestsDispatcher) retrieveLegacyResourceTrackers(ctx context.Cont } for _, rt := range rtList.Items { if rt.Name != a.currentRTName && - (a.previousRT != nil && rt.Name != a.previousRT.Name) && !IsLifeLongResourceTracker(rt) { + (a.previousRT != nil && rt.Name != a.previousRT.Name) && !rt.IsLifeLong() { a.legacyRTs = append(a.legacyRTs, rt.DeepCopy()) } } @@ -222,7 +219,7 @@ func (a *AppManifestsDispatcher) retrieveLegacyResourceTrackers(ctx context.Cont if len(oldRtList.Items) != 0 { for _, rt := range oldRtList.Items { if rt.Name != a.currentRTName && - (a.previousRT != nil && rt.Name != a.previousRT.Name) && !IsLifeLongResourceTracker(rt) { + (a.previousRT != nil && rt.Name != a.previousRT.Name) && !rt.IsLifeLong() { a.legacyRTs = append(a.legacyRTs, rt.DeepCopy()) } } @@ -232,36 +229,13 @@ func (a *AppManifestsDispatcher) retrieveLegacyResourceTrackers(ctx context.Cont } func (a *AppManifestsDispatcher) applyAndRecordManifests(ctx context.Context, manifests []*unstructured.Unstructured) error { - ctrlUIDs := []types.UID{a.currentRT.UID} - if a.previousRT != nil && a.previousRT.Name != a.currentRTName { - klog.InfoS("Going to apply or upgrade resources", "from", a.previousRT.Name, "to", a.currentRTName) - // if two RT's names are different, it means dispatching operation happens in an upgrade or rollout scenario - // in such two scenarios, for those unchanged manifests, we will - // - make sure existing resources are controlled by any of these two resource trackers - // - set new resource tracker as their controller owner - ctrlUIDs = append(ctrlUIDs, a.previousRT.UID) - } - - // allow to apply changes to resources owned by legacy RTs - for _, rt := range a.legacyRTs { - ctrlUIDs = append(ctrlUIDs, rt.UID) - } - - applyOpts := []apply.ApplyOption{apply.MustBeControllableByAny(ctrlUIDs), apply.NotUpdateRenderHashEqual()} - ownerRef := metav1.OwnerReference{ - APIVersion: v1beta1.SchemeGroupVersion.String(), - Kind: reflect.TypeOf(v1beta1.ResourceTracker{}).Name(), - Name: a.currentRT.Name, - UID: a.currentRT.UID, - Controller: pointer.BoolPtr(true), - BlockOwnerDeletion: pointer.BoolPtr(true), - } + applyOpts := []apply.ApplyOption{apply.MustBeControlledByApp(&a.appRev.Spec.Application), apply.NotUpdateRenderHashEqual()} for _, rsc := range manifests { if rsc == nil { continue } - immutable, err := a.ImmutableResourcesUpdate(ctx, rsc, ownerRef, applyOpts) + immutable, err := a.ImmutableResourcesUpdate(ctx, rsc, a.currentRT, applyOpts) if immutable { if err != nil { klog.ErrorS(err, "Failed to apply immutable resource with new ownerReference", "object", @@ -273,7 +247,7 @@ func (a *AppManifestsDispatcher) applyAndRecordManifests(ctx context.Context, ma } // each resource applied by dispatcher MUST be controlled by resource tracker - setOrOverrideOAMControllerOwner(rsc, ownerRef) + a.currentRT.AddOwnerReferenceToTrackerResource(rsc) if err := a.applicator.Apply(ctx, rsc, applyOpts...); err != nil { klog.ErrorS(err, "Failed to apply a resource", "object", klog.KObj(rsc), "apiVersion", rsc.GetAPIVersion(), "kind", rsc.GetKind()) @@ -288,7 +262,7 @@ func (a *AppManifestsDispatcher) applyAndRecordManifests(ctx context.Context, ma // ImmutableResourcesUpdate only updates the ownerReference // TODO(wonderflow): we should allow special fields to be updated. e.g. the resources.requests for bound claims for PV should be able to update -func (a *AppManifestsDispatcher) ImmutableResourcesUpdate(ctx context.Context, res *unstructured.Unstructured, ownerRef metav1.OwnerReference, applyOpts []apply.ApplyOption) (bool, error) { +func (a *AppManifestsDispatcher) ImmutableResourcesUpdate(ctx context.Context, res *unstructured.Unstructured, rt *v1beta1.ResourceTracker, applyOpts []apply.ApplyOption) (bool, error) { if res == nil { return false, nil } @@ -302,7 +276,7 @@ func (a *AppManifestsDispatcher) ImmutableResourcesUpdate(ctx context.Context, r if err != nil { return true, err } - setOrOverrideOAMControllerOwner(pv, ownerRef) + rt.AddOwnerReferenceToTrackerResource(pv) pv.SetGroupVersionKind(v1.SchemeGroupVersion.WithKind(reflect.TypeOf(v1.PersistentVolume{}).Name())) return true, a.applicator.Apply(ctx, pv, applyOpts...) default: @@ -313,29 +287,13 @@ func (a *AppManifestsDispatcher) ImmutableResourcesUpdate(ctx context.Context, r func (a *AppManifestsDispatcher) updateResourceTrackerStatus(ctx context.Context, appliedManifests []*unstructured.Unstructured) error { // merge applied resources and already tracked ones if a.currentRT.Status.TrackedResources == nil { - a.currentRT.Status.TrackedResources = make([]v1.ObjectReference, 0) + a.currentRT.Status.TrackedResources = make([]common.ClusterObjectReference, 0) } for _, rsc := range appliedManifests { if rsc == nil { continue } - appliedRef := v1.ObjectReference{ - APIVersion: rsc.GetAPIVersion(), - Kind: rsc.GetKind(), - Name: rsc.GetName(), - Namespace: rsc.GetNamespace(), - } - alreadyTracked := false - for _, tracked := range a.currentRT.Status.TrackedResources { - if tracked.APIVersion == appliedRef.APIVersion && tracked.Kind == appliedRef.Kind && - tracked.Name == appliedRef.Name && tracked.Namespace == appliedRef.Namespace { - alreadyTracked = true - break - } - } - if !alreadyTracked { - a.currentRT.Status.TrackedResources = append(a.currentRT.Status.TrackedResources, appliedRef) - } + a.currentRT.AddTrackedResource(rsc) } // TODO move TrackedResources from status to spec @@ -355,36 +313,3 @@ func (a *AppManifestsDispatcher) updateResourceTrackerStatus(ctx context.Context klog.InfoS("Successfully update resource tracker status", "resourceTracker", a.currentRTName) return nil } - -// ObjectOwner is a interface for get and set ownerReference -type ObjectOwner interface { - GetOwnerReferences() []metav1.OwnerReference - SetOwnerReferences([]metav1.OwnerReference) -} - -// setOrOverrideOAMControllerOwner will set the new owner and remove the legacy OAM owner -func setOrOverrideOAMControllerOwner(obj ObjectOwner, controllerOwner metav1.OwnerReference) { - newOwnerRefs := []metav1.OwnerReference{controllerOwner} - for _, owner := range obj.GetOwnerReferences() { - // delete the old resourceTracker owner - if owner.Kind == v1beta1.ResourceTrackerKind && owner.APIVersion == v1beta1.SchemeGroupVersion.String() { - continue - } - // delete the old appContext owner - if owner.Kind == "ApplicationContext" && owner.APIVersion == v1alpha2.SchemeGroupVersion.String() { - continue - } - if owner.Controller != nil && *owner.Controller && - owner.UID != controllerOwner.UID { - owner.Controller = pointer.BoolPtr(false) - } - newOwnerRefs = append(newOwnerRefs, owner) - } - obj.SetOwnerReferences(newOwnerRefs) -} - -// IsLifeLongResourceTracker check if resourcetracker shares the same whole life with the entire application -func IsLifeLongResourceTracker(rt v1beta1.ResourceTracker) bool { - _, ok := rt.GetAnnotations()[oam.AnnotationResourceTrackerLifeLong] - return ok -} diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/dispatch_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/dispatch_test.go index 8a233e6ec..ef4fef142 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/dispatch_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/dispatch_test.go @@ -22,14 +22,17 @@ import ( "github.com/stretchr/testify/assert" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/utils/pointer" "github.com/oam-dev/kubevela/apis/core.oam.dev/common" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/apis/interfaces" "github.com/oam-dev/kubevela/pkg/oam" ) func TestSetOAMOwner(t *testing.T) { tests := map[string]struct { - OO ObjectOwner + OO interfaces.ObjectOwner CO v1.OwnerReference ExpOwner []v1.OwnerReference }{ @@ -41,37 +44,11 @@ func TestSetOAMOwner(t *testing.T) { Name: "myapp", }, ExpOwner: []v1.OwnerReference{{ - APIVersion: "core.oam.dev/v1beta1", - Kind: "ResourceTracker", - Name: "myapp", - }}, - }, - "test remove old resourceTracker owner": { - OO: &unstructured.Unstructured{ - Object: map[string]interface{}{ - "metadata": map[string]interface{}{ - "ownerReferences": []interface{}{ - map[string]interface{}{ - "apiVersion": "core.oam.dev/v1beta1", - "kind": "ResourceTracker", - }, - map[string]interface{}{ - "apiVersion": "core.oam.dev/v1alpha2", - "kind": "ApplicationContext", - }, - }, - }, - }, - }, - CO: v1.OwnerReference{ - APIVersion: "core.oam.dev/v1beta1", - Kind: "ResourceTracker", - Name: "myapp", - }, - ExpOwner: []v1.OwnerReference{{ - APIVersion: "core.oam.dev/v1beta1", - Kind: "ResourceTracker", - Name: "myapp", + APIVersion: "core.oam.dev/v1beta1", + Kind: "ResourceTracker", + Name: "myapp", + Controller: pointer.Bool(true), + BlockOwnerDeletion: pointer.Bool(true), }}, }, "test other owner not removed": { @@ -94,19 +71,22 @@ func TestSetOAMOwner(t *testing.T) { Name: "myapp", }, ExpOwner: []v1.OwnerReference{{ - APIVersion: "core.oam.dev/v1beta1", - Kind: "ResourceTracker", - Name: "myapp", - }, - { - APIVersion: "core.oam.dev/v1alpha1", - Kind: "Rollout", - Name: "xxx", - }}, + APIVersion: "core.oam.dev/v1beta1", + Kind: "ResourceTracker", + Name: "myapp", + Controller: pointer.Bool(true), + BlockOwnerDeletion: pointer.Bool(true), + }, { + APIVersion: "core.oam.dev/v1alpha1", + Kind: "Rollout", + Name: "xxx", + }}, }, } for name, ti := range tests { - setOrOverrideOAMControllerOwner(ti.OO, ti.CO) + rt := &v1beta1.ResourceTracker{} + rt.Name = ti.CO.Name + rt.AddOwnerReferenceToTrackerResource(ti.OO) assert.Equal(t, ti.ExpOwner, ti.OO.GetOwnerReferences(), name) } } diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/gc_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/gc_test.go index bcaff9018..f92f3ff74 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/gc_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/dispatch/gc_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + "github.com/oam-dev/kubevela/apis/core.oam.dev/common" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" ) @@ -33,31 +34,39 @@ func TestIsTrackedResources(t *testing.T) { }{{ oldRT: &v1beta1.ResourceTracker{ Status: v1beta1.ResourceTrackerStatus{ - TrackedResources: []corev1.ObjectReference{{ - Kind: "Deployment", - APIVersion: "apps/v1", - Name: "test", - Namespace: "default", + TrackedResources: []common.ClusterObjectReference{{ + ObjectReference: corev1.ObjectReference{ + Kind: "Deployment", + APIVersion: "apps/v1", + Name: "test", + Namespace: "default", + }, }, { - Kind: "Pod", - APIVersion: "v1", - Name: "test", - Namespace: "default", + ObjectReference: corev1.ObjectReference{ + Kind: "Pod", + APIVersion: "v1", + Name: "test", + Namespace: "default", + }, }}, }, }, newRT: &v1beta1.ResourceTracker{ Status: v1beta1.ResourceTrackerStatus{ - TrackedResources: []corev1.ObjectReference{{ - Kind: "Deployment", - APIVersion: "apps/v1", - Name: "test", - Namespace: "default", + TrackedResources: []common.ClusterObjectReference{{ + ObjectReference: corev1.ObjectReference{ + Kind: "Deployment", + APIVersion: "apps/v1", + Name: "test", + Namespace: "default", + }, }, { - Kind: "Pod", - APIVersion: "v1", - Name: "test", - Namespace: "default", + ObjectReference: corev1.ObjectReference{ + Kind: "Pod", + APIVersion: "v1", + Name: "test", + Namespace: "default", + }, }}, }, }, @@ -65,31 +74,39 @@ func TestIsTrackedResources(t *testing.T) { }, { oldRT: &v1beta1.ResourceTracker{ Status: v1beta1.ResourceTrackerStatus{ - TrackedResources: []corev1.ObjectReference{{ - Kind: "Deployment", - APIVersion: "apps/v1", - Name: "test", - Namespace: "default", + TrackedResources: []common.ClusterObjectReference{{ + ObjectReference: corev1.ObjectReference{ + Kind: "Deployment", + APIVersion: "apps/v1", + Name: "test", + Namespace: "default", + }, }, { - Kind: "Pod", - APIVersion: "v1", - Name: "hello", - Namespace: "default", + ObjectReference: corev1.ObjectReference{ + Kind: "Pod", + APIVersion: "v1", + Name: "hello", + Namespace: "default", + }, }}, }, }, newRT: &v1beta1.ResourceTracker{ Status: v1beta1.ResourceTrackerStatus{ - TrackedResources: []corev1.ObjectReference{{ - Kind: "Deployment", - APIVersion: "apps/v1", - Name: "test", - Namespace: "default", + TrackedResources: []common.ClusterObjectReference{{ + ObjectReference: corev1.ObjectReference{ + Kind: "Deployment", + APIVersion: "apps/v1", + Name: "test", + Namespace: "default", + }, }, { - Kind: "Pod", - APIVersion: "v1", - Name: "test", - Namespace: "default", + ObjectReference: corev1.ObjectReference{ + Kind: "Pod", + APIVersion: "v1", + Name: "test", + Namespace: "default", + }, }}, }, }, @@ -98,16 +115,20 @@ func TestIsTrackedResources(t *testing.T) { oldRT: &v1beta1.ResourceTracker{}, newRT: &v1beta1.ResourceTracker{ Status: v1beta1.ResourceTrackerStatus{ - TrackedResources: []corev1.ObjectReference{{ - Kind: "Deployment", - APIVersion: "apps/v1", - Name: "test", - Namespace: "default", + TrackedResources: []common.ClusterObjectReference{{ + ObjectReference: corev1.ObjectReference{ + Kind: "Deployment", + APIVersion: "apps/v1", + Name: "test", + Namespace: "default", + }, }, { - Kind: "Pod", - APIVersion: "v1", - Name: "test", - Namespace: "default", + ObjectReference: corev1.ObjectReference{ + Kind: "Pod", + APIVersion: "v1", + Name: "test", + Namespace: "default", + }, }}, }, }, @@ -115,16 +136,20 @@ func TestIsTrackedResources(t *testing.T) { }, { oldRT: &v1beta1.ResourceTracker{ Status: v1beta1.ResourceTrackerStatus{ - TrackedResources: []corev1.ObjectReference{{ - Kind: "Deployment", - APIVersion: "apps/v1", - Name: "test", - Namespace: "default", + TrackedResources: []common.ClusterObjectReference{{ + ObjectReference: corev1.ObjectReference{ + Kind: "Deployment", + APIVersion: "apps/v1", + Name: "test", + Namespace: "default", + }, }, { - Kind: "Pod", - APIVersion: "v1", - Name: "test", - Namespace: "default", + ObjectReference: corev1.ObjectReference{ + Kind: "Pod", + APIVersion: "v1", + Name: "test", + Namespace: "default", + }, }}, }, }, diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/revision.go b/pkg/controller/core.oam.dev/v1alpha2/application/revision.go index 87888b5de..2a9a0681d 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/revision.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/revision.go @@ -25,6 +25,7 @@ import ( "github.com/oam-dev/kubevela/pkg/cue/model" "github.com/oam-dev/kubevela/pkg/multicluster" + "github.com/oam-dev/kubevela/pkg/resourcetracker" "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" @@ -598,27 +599,6 @@ func ComputeComponentRevisionHash(comp *types.ComponentManifest) (string, error) return utils.ComputeSpecHash(&compRevisionHash) } -// createOrGetResourceTracker create or get a resource tracker to manage all componentRevisions -func (h *AppHandler) createOrGetResourceTracker(ctx context.Context) (*v1beta1.ResourceTracker, error) { - rt := &v1beta1.ResourceTracker{} - rtName := h.app.Name + "-" + h.app.Namespace - if err := h.r.Get(ctx, ktypes.NamespacedName{Name: rtName}, rt); err != nil { - if !apierrors.IsNotFound(err) { - return nil, err - } - rt.SetName(rtName) - rt.SetLabels(map[string]string{ - oam.LabelAppName: h.app.Name, - oam.LabelAppNamespace: h.app.Namespace, - }) - rt.SetAnnotations(map[string]string{oam.AnnotationResourceTrackerLifeLong: "true"}) - if err = h.r.Create(ctx, rt); err != nil { - return nil, err - } - } - return rt, nil -} - // createControllerRevision records snapshot of a component func (h *AppHandler) createControllerRevision(ctx context.Context, cm *types.ComponentManifest) error { comp, err := componentManifest2Component(cm) @@ -626,7 +606,7 @@ func (h *AppHandler) createControllerRevision(ctx context.Context, cm *types.Com return err } revision, _ := utils.ExtractRevision(cm.RevisionName) - rt, err := h.createOrGetResourceTracker(ctx) + rt, err := resourcetracker.CreateOrGetApplicationRootResourceTracker(ctx, h.r.Client, h.app) if err != nil { return err } @@ -798,7 +778,7 @@ func gatherUsingAppRevision(ctx context.Context, h *AppHandler) (map[string]bool return nil, err } for _, rt := range rtList.Items { - if dispatch.IsLifeLongResourceTracker(rt) { + if rt.IsLifeLong() { continue } appRev := dispatch.ExtractAppRevisionName(rt.Name, ns) diff --git a/pkg/controller/standard.oam.dev/v1alpha1/rollout/handler.go b/pkg/controller/standard.oam.dev/v1alpha1/rollout/handler.go index c32f1c900..ad309cb89 100644 --- a/pkg/controller/standard.oam.dev/v1alpha1/rollout/handler.go +++ b/pkg/controller/standard.oam.dev/v1alpha1/rollout/handler.go @@ -25,7 +25,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -317,14 +316,7 @@ func (h *handler) recordWorkloadInResourceTracker(ctx context.Context, workload klog.Errorf("fail to get resourceTracker to record workload rollout: namespace:%s, name: %s", h.rollout.Namespace, h.rollout.Name) return err } - recordedWorkload := corev1.ObjectReference{ - APIVersion: workload.GetAPIVersion(), - Kind: workload.GetKind(), - UID: workload.GetUID(), - Namespace: workload.GetNamespace(), - Name: workload.GetName(), - } - rt.Status.TrackedResources = append(rt.Status.TrackedResources, recordedWorkload) + rt.AddTrackedResource(workload) if err := h.Status().Update(ctx, &rt); err != nil { klog.Errorf("fail to update resourceTracker for rollout record workload namespace:%s, name: %s", h.rollout.Namespace, h.rollout.Name) return err diff --git a/pkg/multicluster/gc.go b/pkg/multicluster/gc.go index 5853ca15d..a1c764726 100644 --- a/pkg/multicluster/gc.go +++ b/pkg/multicluster/gc.go @@ -27,24 +27,36 @@ import ( "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/pkg/oam" "github.com/oam-dev/kubevela/pkg/policy/envbinding" + "github.com/oam-dev/kubevela/pkg/resourcetracker" errors2 "github.com/oam-dev/kubevela/pkg/utils/errors" ) -func getAppliedClusters(app *v1beta1.Application) []string { +func getClustersFromRootResourceTracker(ctx context.Context, c client.Client, app *v1beta1.Application) []string { + rt, err := resourcetracker.GetApplicationRootResourceTracker(ctx, c, app) + if err != nil { + return nil + } + return rt.GetTrackedClusters() +} + +func getAppliedClusters(ctx context.Context, c client.Client, app *v1beta1.Application) []string { + appliedClusters := map[string]bool{} + for _, v := range app.Status.AppliedResources { + appliedClusters[v.Cluster] = true + } status, err := envbinding.GetEnvBindingPolicyStatus(app, "") - appliedClusters := map[string]bool{} if err != nil { - klog.InfoS("failed to get envbinding policy status during gc", "err", err.Error()) // fallback - for _, v := range app.Status.AppliedResources { - appliedClusters[v.Cluster] = true - } + klog.InfoS("failed to get envbinding policy status during gc", "err", err.Error()) } if status != nil { for _, conn := range status.ClusterConnections { appliedClusters[conn.ClusterName] = true } } + for _, cluster := range getClustersFromRootResourceTracker(ctx, c, app) { + appliedClusters[cluster] = true + } var clusters []string for cluster := range appliedClusters { clusters = append(clusters, cluster) @@ -53,9 +65,9 @@ func getAppliedClusters(app *v1beta1.Application) []string { } // GarbageCollectionForOutdatedResourcesInSubClusters run garbage collection in sub clusters and remove outdated ResourceTrackers with their associated resources -func GarbageCollectionForOutdatedResourcesInSubClusters(ctx context.Context, app *v1beta1.Application, gcHandler func(context.Context) error) error { +func GarbageCollectionForOutdatedResourcesInSubClusters(ctx context.Context, c client.Client, app *v1beta1.Application, gcHandler func(context.Context) error) error { var errs errors2.ErrorList - for _, clusterName := range getAppliedClusters(app) { + for _, clusterName := range getAppliedClusters(ctx, c, app) { if err := gcHandler(ContextWithClusterName(ctx, clusterName)); err != nil { if !errors.As(err, &errors2.ResourceTrackerNotExistError{}) { errs.Append(errors.Wrapf(err, "failed to run gc in subCluster %s", clusterName)) @@ -68,27 +80,44 @@ func GarbageCollectionForOutdatedResourcesInSubClusters(ctx context.Context, app return nil } -// GarbageCollectionForAllResourceTrackersInSubCluster run garbage collection in sub clusters and remove all ResourceTrackers for the EnvBinding -func GarbageCollectionForAllResourceTrackersInSubCluster(ctx context.Context, c client.Client, app *v1beta1.Application) error { - // delete subCluster resourceTracker - for _, cluster := range getAppliedClusters(app) { - subCtx := ContextWithClusterName(ctx, cluster) - listOpts := []client.ListOption{ - client.MatchingLabels{ - oam.LabelAppName: app.Name, - oam.LabelAppNamespace: app.Namespace, - }} - rtList := &v1beta1.ResourceTrackerList{} - if err := c.List(subCtx, rtList, listOpts...); err != nil { - klog.ErrorS(err, "failed to list resource tracker of app", "name", app.Name, "cluster", cluster) +func garbageCollectResourceTrackers(ctx context.Context, c client.Client, app *v1beta1.Application, cluster string) error { + if cluster != "" { + ctx = ContextWithClusterName(ctx, cluster) + } + listOpts := []client.ListOption{ + client.MatchingLabels{ + oam.LabelAppName: app.Name, + oam.LabelAppNamespace: app.Namespace, + }} + rtList := &v1beta1.ResourceTrackerList{} + if err := c.List(ctx, rtList, listOpts...); err != nil { + klog.ErrorS(err, "failed to list resource tracker of app", "name", app.Name, "cluster", cluster) + return errors.WithMessage(err, "cannot remove finalizer") + } + for _, rt := range rtList.Items { + if err := c.Delete(ctx, rt.DeepCopy()); err != nil && !kerrors.IsNotFound(err) { + klog.ErrorS(err, "failed to delete resource tracker", "name", rt.Name) return errors.WithMessage(err, "cannot remove finalizer") } - for _, rt := range rtList.Items { - if err := c.Delete(subCtx, rt.DeepCopy()); err != nil && !kerrors.IsNotFound(err) { - klog.ErrorS(err, "failed to delete resource tracker", "name", rt.Name) - return errors.WithMessage(err, "cannot remove finalizer") - } - } } return nil } + +// GarbageCollectionForAllResourceTrackersInSubCluster run garbage collection in sub clusters and remove all ResourceTrackers +func GarbageCollectionForAllResourceTrackersInSubCluster(ctx context.Context, c client.Client, app *v1beta1.Application) error { + // delete subCluster resourceTracker + for _, cluster := range getAppliedClusters(ctx, c, app) { + if err := garbageCollectResourceTrackers(ctx, c, app, cluster); err != nil { + return err + } + } + return nil +} + +// GarbageCollectionForAllResourceTrackers run garbage collection in sub clusters and remove all ResourceTrackers, including managed cluster +func GarbageCollectionForAllResourceTrackers(ctx context.Context, c client.Client, app *v1beta1.Application) error { + if err := GarbageCollectionForAllResourceTrackersInSubCluster(ctx, c, app); err != nil { + return err + } + return garbageCollectResourceTrackers(ctx, c, app, "") +} diff --git a/pkg/multicluster/gc_test.go b/pkg/multicluster/gc_test.go index f47097c00..3e7a74ecc 100644 --- a/pkg/multicluster/gc_test.go +++ b/pkg/multicluster/gc_test.go @@ -17,16 +17,19 @@ limitations under the License. package multicluster import ( + "context" "encoding/json" "sort" "testing" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/oam-dev/kubevela/apis/core.oam.dev/common" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha1" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + common2 "github.com/oam-dev/kubevela/pkg/utils/common" ) func TestGetAppliedCluster(t *testing.T) { @@ -35,11 +38,12 @@ func TestGetAppliedCluster(t *testing.T) { app.Status.AppliedResources = []common.ClusterObjectReference{{ Cluster: "cluster-0", }} + cli := fake.NewClientBuilder().WithScheme(common2.Scheme).Build() app.Status.PolicyStatus = []common.PolicyStatus{{ Type: v1alpha1.EnvBindingPolicyType, Status: &runtime.RawExtension{Raw: []byte(`bad value`)}, }} - clusters := getAppliedClusters(app) + clusters := getAppliedClusters(context.Background(), cli, app) r.Equal(1, len(clusters)) r.Equal("cluster-0", clusters[0]) envBindingStatus := &v1alpha1.EnvBindingStatus{ClusterConnections: []v1alpha1.ClusterConnection{{ @@ -49,11 +53,12 @@ func TestGetAppliedCluster(t *testing.T) { }}} bs, err := json.Marshal(envBindingStatus) r.NoError(err) + app.Status.AppliedResources = []common.ClusterObjectReference{} app.Status.PolicyStatus = []common.PolicyStatus{{ Type: v1alpha1.EnvBindingPolicyType, Status: &runtime.RawExtension{Raw: bs}, }} - clusters = getAppliedClusters(app) + clusters = getAppliedClusters(context.Background(), cli, app) r.Equal(2, len(clusters)) sort.Strings(clusters) r.Equal("cluster-1", clusters[0]) diff --git a/pkg/policy/envbinding/placement.go b/pkg/policy/envbinding/placement.go index bd09c4e2c..0539d31cc 100644 --- a/pkg/policy/envbinding/placement.go +++ b/pkg/policy/envbinding/placement.go @@ -17,13 +17,17 @@ limitations under the License. package envbinding import ( + "context" "encoding/json" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/oam-dev/kubevela/apis/core.oam.dev/common" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha1" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/pkg/resourcetracker" ) // ReadPlacementDecisions read placement decisions from application status, return (decisions, if decision is made, error) @@ -123,3 +127,23 @@ func WritePlacementDecisions(app *v1beta1.Application, policyName string, envNam } return nil } + +// WriteClustersToResourceTracker records cluster into application root resource tracker +func WriteClustersToResourceTracker(ctx context.Context, c client.Client, app *v1beta1.Application, clusters ...string) error { + rt, err := resourcetracker.CreateOrGetApplicationRootResourceTracker(ctx, c, app) + if err != nil { + return errors.Wrapf(err, "failed to create/get application root resource tracker") + } + update := false + for _, cluster := range clusters { + if exists := rt.AddTrackedCluster(cluster); !exists { + update = true + } + } + if update { + if err = c.Status().Update(ctx, rt); err != nil { + return errors.Wrapf(err, "failed to add cluster into application root resource tracker") + } + } + return nil +} diff --git a/pkg/resourcetracker/resourcetracker.go b/pkg/resourcetracker/resourcetracker.go new file mode 100644 index 000000000..e9b550551 --- /dev/null +++ b/pkg/resourcetracker/resourcetracker.go @@ -0,0 +1,62 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcetracker + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/pkg/oam" +) + +// GetApplicationRootResourceTracker get root resourcetracker for application +// root resourcetracker if a life-long resourcetracker which shares the life-cycle with the application instead of one revision +func GetApplicationRootResourceTracker(ctx context.Context, c client.Client, app *v1beta1.Application) (*v1beta1.ResourceTracker, error) { + rt := &v1beta1.ResourceTracker{} + rtName := app.Name + "-" + app.Namespace + if err := c.Get(ctx, types.NamespacedName{Name: rtName}, rt); err != nil { + return nil, err + } + return rt, nil +} + +// CreateOrGetApplicationRootResourceTracker create or get root resourcetracker for application +// root resourcetracker if a life-long resourcetracker which shares the life-cycle with the application instead of one revision +func CreateOrGetApplicationRootResourceTracker(ctx context.Context, c client.Client, app *v1beta1.Application) (*v1beta1.ResourceTracker, error) { + rt, err := GetApplicationRootResourceTracker(ctx, c, app) + if err != nil { + if !errors.IsNotFound(err) { + return nil, err + } + rt = &v1beta1.ResourceTracker{} + rtName := app.Name + "-" + app.Namespace + rt.SetName(rtName) + rt.SetLabels(map[string]string{ + oam.LabelAppName: app.Name, + oam.LabelAppNamespace: app.Namespace, + }) + rt.SetLifeLong() + if err = c.Create(ctx, rt); err != nil { + return nil, err + } + } + return rt, nil +} diff --git a/pkg/utils/apply/apply.go b/pkg/utils/apply/apply.go index 1f8b0ca00..72d2b7791 100644 --- a/pkg/utils/apply/apply.go +++ b/pkg/utils/apply/apply.go @@ -18,7 +18,10 @@ package apply import ( "context" + "fmt" + "regexp" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/pkg/controller/utils" "github.com/oam-dev/kubevela/pkg/oam/util" @@ -273,6 +276,37 @@ func MustBeControllableByAny(ctrlUIDs []types.UID) ApplyOption { } } +// MustBeControlledByApp requires that the new object is controllable by versioned resourcetracker +func MustBeControlledByApp(app *v1beta1.Application) ApplyOption { + pattern := "^" + app.GetName() + "-v[0-9]+-" + app.GetNamespace() + "$" + return func(_ *applyAction, existing, _ client.Object) error { + if existing == nil { + return nil + } + existingObjMeta, _ := existing.(metav1.Object) + c := metav1.GetControllerOf(existingObjMeta) + if c == nil { + return nil + } + + // NOTE This is for backward compatibility after ApplicationContext is deprecated. + // In legacy clusters, existing resources are ctrl-owned by ApplicationContext or ResourceTracker (only for + // cx-namespace and cluster-scope resources). We use a particular annotation to identify legacy resources. + if len(existingObjMeta.GetAnnotations()[oam.AnnotationKubeVelaVersion]) == 0 { + // just skip checking UIDs, '3-way-merge' will remove the legacy ctrl-owner automatically + return nil + } + + if !(c.APIVersion == v1beta1.SchemeGroupVersion.String()) || !(c.Kind == v1beta1.ResourceTrackerKind) { + return fmt.Errorf("existing object is not controlled by resourcetracker, currently controlled by %s[%s]", c.Kind, c.APIVersion) + } + if !regexp.MustCompile(pattern).MatchString(c.Name) { + return fmt.Errorf("existing object is controlled by resourcetracker %s which does not conform to pattern %s", c.Name, pattern) + } + return nil + } +} + // MakeCustomApplyOption let user can generate applyOption that restrict change apply action. func MakeCustomApplyOption(f func(existing, desired client.Object) error) ApplyOption { return func(act *applyAction, existing, desired client.Object) error { diff --git a/pkg/workflow/providers/multicluster/multicluster.go b/pkg/workflow/providers/multicluster/multicluster.go index 39dd45995..8eeb49676 100644 --- a/pkg/workflow/providers/multicluster/multicluster.go +++ b/pkg/workflow/providers/multicluster/multicluster.go @@ -115,6 +115,9 @@ func (p *provider) MakePlacementDecisions(ctx wfContext.Context, v *value.Value, if err = envbinding.WritePlacementDecisions(p.app, policy, env, decisions); err != nil { return err } + if err = envbinding.WriteClustersToResourceTracker(context.Background(), p.Client, p.app, clusterName); err != nil { + return err + } return v.FillObject(map[string]interface{}{"decisions": decisions}, "outputs") }