Feat: rework rt (#2797)

This commit is contained in:
Somefive
2021-11-29 16:14:30 +08:00
committed by GitHub
parent d1f56743cc
commit f9ecc02417
20 changed files with 511 additions and 367 deletions
@@ -219,7 +219,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
if wfStatus != nil {
ref, err := handler.DispatchAndGC(ctx)
if err == nil {
err = multicluster.GarbageCollectionForOutdatedResourcesInSubClusters(ctx, app, func(c context.Context) error {
err = multicluster.GarbageCollectionForOutdatedResourcesInSubClusters(ctx, r.Client, app, func(c context.Context) error {
_, e := handler.DispatchAndGC(c)
return e
})
@@ -332,23 +332,7 @@ func (r *Reconciler) handleFinalizers(ctx monitorContext.Context, app *v1beta1.A
return true, errors.Wrap(r.Client.Update(ctx, app), errUpdateApplicationFinalizer)
}
if meta.FinalizerExists(app, resourceTrackerFinalizer) || meta.FinalizerExists(app, legacyOnlyRevisionFinalizer) {
listOpts := []client.ListOption{
client.MatchingLabels{
oam.LabelAppName: app.Name,
oam.LabelAppNamespace: app.Namespace,
}}
rtList := &v1beta1.ResourceTrackerList{}
if err := r.Client.List(ctx, rtList, listOpts...); err != nil {
ctx.Error(err, "Failed to list resource tracker of app", "name", app.Name)
return true, errors.WithMessage(err, "cannot remove finalizer")
}
for _, rt := range rtList.Items {
if err := r.Client.Delete(ctx, rt.DeepCopy()); err != nil && !kerrors.IsNotFound(err) {
ctx.Error(err, "Failed to delete resource tracker", "name", rt.Name)
return true, errors.WithMessage(err, "cannot remove finalizer")
}
}
if err := multicluster.GarbageCollectionForAllResourceTrackersInSubCluster(ctx, r.Client, app); err != nil {
if err := multicluster.GarbageCollectionForAllResourceTrackers(ctx, r.Client, app); err != nil {
return true, err
}
meta.RemoveFinalizer(app, resourceTrackerFinalizer)
@@ -23,15 +23,12 @@ import (
"github.com/pkg/errors"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
"k8s.io/klog/v2"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2"
"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
"github.com/oam-dev/kubevela/pkg/oam"
"github.com/oam-dev/kubevela/pkg/utils/apply"
@@ -205,7 +202,7 @@ func (a *AppManifestsDispatcher) retrieveLegacyResourceTrackers(ctx context.Cont
}
for _, rt := range rtList.Items {
if rt.Name != a.currentRTName &&
(a.previousRT != nil && rt.Name != a.previousRT.Name) && !IsLifeLongResourceTracker(rt) {
(a.previousRT != nil && rt.Name != a.previousRT.Name) && !rt.IsLifeLong() {
a.legacyRTs = append(a.legacyRTs, rt.DeepCopy())
}
}
@@ -222,7 +219,7 @@ func (a *AppManifestsDispatcher) retrieveLegacyResourceTrackers(ctx context.Cont
if len(oldRtList.Items) != 0 {
for _, rt := range oldRtList.Items {
if rt.Name != a.currentRTName &&
(a.previousRT != nil && rt.Name != a.previousRT.Name) && !IsLifeLongResourceTracker(rt) {
(a.previousRT != nil && rt.Name != a.previousRT.Name) && !rt.IsLifeLong() {
a.legacyRTs = append(a.legacyRTs, rt.DeepCopy())
}
}
@@ -232,36 +229,13 @@ func (a *AppManifestsDispatcher) retrieveLegacyResourceTrackers(ctx context.Cont
}
func (a *AppManifestsDispatcher) applyAndRecordManifests(ctx context.Context, manifests []*unstructured.Unstructured) error {
ctrlUIDs := []types.UID{a.currentRT.UID}
if a.previousRT != nil && a.previousRT.Name != a.currentRTName {
klog.InfoS("Going to apply or upgrade resources", "from", a.previousRT.Name, "to", a.currentRTName)
// if two RT's names are different, it means dispatching operation happens in an upgrade or rollout scenario
// in such two scenarios, for those unchanged manifests, we will
// - make sure existing resources are controlled by any of these two resource trackers
// - set new resource tracker as their controller owner
ctrlUIDs = append(ctrlUIDs, a.previousRT.UID)
}
// allow to apply changes to resources owned by legacy RTs
for _, rt := range a.legacyRTs {
ctrlUIDs = append(ctrlUIDs, rt.UID)
}
applyOpts := []apply.ApplyOption{apply.MustBeControllableByAny(ctrlUIDs), apply.NotUpdateRenderHashEqual()}
ownerRef := metav1.OwnerReference{
APIVersion: v1beta1.SchemeGroupVersion.String(),
Kind: reflect.TypeOf(v1beta1.ResourceTracker{}).Name(),
Name: a.currentRT.Name,
UID: a.currentRT.UID,
Controller: pointer.BoolPtr(true),
BlockOwnerDeletion: pointer.BoolPtr(true),
}
applyOpts := []apply.ApplyOption{apply.MustBeControlledByApp(&a.appRev.Spec.Application), apply.NotUpdateRenderHashEqual()}
for _, rsc := range manifests {
if rsc == nil {
continue
}
immutable, err := a.ImmutableResourcesUpdate(ctx, rsc, ownerRef, applyOpts)
immutable, err := a.ImmutableResourcesUpdate(ctx, rsc, a.currentRT, applyOpts)
if immutable {
if err != nil {
klog.ErrorS(err, "Failed to apply immutable resource with new ownerReference", "object",
@@ -273,7 +247,7 @@ func (a *AppManifestsDispatcher) applyAndRecordManifests(ctx context.Context, ma
}
// each resource applied by dispatcher MUST be controlled by resource tracker
setOrOverrideOAMControllerOwner(rsc, ownerRef)
a.currentRT.AddOwnerReferenceToTrackerResource(rsc)
if err := a.applicator.Apply(ctx, rsc, applyOpts...); err != nil {
klog.ErrorS(err, "Failed to apply a resource", "object",
klog.KObj(rsc), "apiVersion", rsc.GetAPIVersion(), "kind", rsc.GetKind())
@@ -288,7 +262,7 @@ func (a *AppManifestsDispatcher) applyAndRecordManifests(ctx context.Context, ma
// ImmutableResourcesUpdate only updates the ownerReference
// TODO(wonderflow): we should allow special fields to be updated. e.g. the resources.requests for bound claims for PV should be able to update
func (a *AppManifestsDispatcher) ImmutableResourcesUpdate(ctx context.Context, res *unstructured.Unstructured, ownerRef metav1.OwnerReference, applyOpts []apply.ApplyOption) (bool, error) {
func (a *AppManifestsDispatcher) ImmutableResourcesUpdate(ctx context.Context, res *unstructured.Unstructured, rt *v1beta1.ResourceTracker, applyOpts []apply.ApplyOption) (bool, error) {
if res == nil {
return false, nil
}
@@ -302,7 +276,7 @@ func (a *AppManifestsDispatcher) ImmutableResourcesUpdate(ctx context.Context, r
if err != nil {
return true, err
}
setOrOverrideOAMControllerOwner(pv, ownerRef)
rt.AddOwnerReferenceToTrackerResource(pv)
pv.SetGroupVersionKind(v1.SchemeGroupVersion.WithKind(reflect.TypeOf(v1.PersistentVolume{}).Name()))
return true, a.applicator.Apply(ctx, pv, applyOpts...)
default:
@@ -313,29 +287,13 @@ func (a *AppManifestsDispatcher) ImmutableResourcesUpdate(ctx context.Context, r
func (a *AppManifestsDispatcher) updateResourceTrackerStatus(ctx context.Context, appliedManifests []*unstructured.Unstructured) error {
// merge applied resources and already tracked ones
if a.currentRT.Status.TrackedResources == nil {
a.currentRT.Status.TrackedResources = make([]v1.ObjectReference, 0)
a.currentRT.Status.TrackedResources = make([]common.ClusterObjectReference, 0)
}
for _, rsc := range appliedManifests {
if rsc == nil {
continue
}
appliedRef := v1.ObjectReference{
APIVersion: rsc.GetAPIVersion(),
Kind: rsc.GetKind(),
Name: rsc.GetName(),
Namespace: rsc.GetNamespace(),
}
alreadyTracked := false
for _, tracked := range a.currentRT.Status.TrackedResources {
if tracked.APIVersion == appliedRef.APIVersion && tracked.Kind == appliedRef.Kind &&
tracked.Name == appliedRef.Name && tracked.Namespace == appliedRef.Namespace {
alreadyTracked = true
break
}
}
if !alreadyTracked {
a.currentRT.Status.TrackedResources = append(a.currentRT.Status.TrackedResources, appliedRef)
}
a.currentRT.AddTrackedResource(rsc)
}
// TODO move TrackedResources from status to spec
@@ -355,36 +313,3 @@ func (a *AppManifestsDispatcher) updateResourceTrackerStatus(ctx context.Context
klog.InfoS("Successfully update resource tracker status", "resourceTracker", a.currentRTName)
return nil
}
// ObjectOwner is a interface for get and set ownerReference
type ObjectOwner interface {
GetOwnerReferences() []metav1.OwnerReference
SetOwnerReferences([]metav1.OwnerReference)
}
// setOrOverrideOAMControllerOwner will set the new owner and remove the legacy OAM owner
func setOrOverrideOAMControllerOwner(obj ObjectOwner, controllerOwner metav1.OwnerReference) {
newOwnerRefs := []metav1.OwnerReference{controllerOwner}
for _, owner := range obj.GetOwnerReferences() {
// delete the old resourceTracker owner
if owner.Kind == v1beta1.ResourceTrackerKind && owner.APIVersion == v1beta1.SchemeGroupVersion.String() {
continue
}
// delete the old appContext owner
if owner.Kind == "ApplicationContext" && owner.APIVersion == v1alpha2.SchemeGroupVersion.String() {
continue
}
if owner.Controller != nil && *owner.Controller &&
owner.UID != controllerOwner.UID {
owner.Controller = pointer.BoolPtr(false)
}
newOwnerRefs = append(newOwnerRefs, owner)
}
obj.SetOwnerReferences(newOwnerRefs)
}
// IsLifeLongResourceTracker check if resourcetracker shares the same whole life with the entire application
func IsLifeLongResourceTracker(rt v1beta1.ResourceTracker) bool {
_, ok := rt.GetAnnotations()[oam.AnnotationResourceTrackerLifeLong]
return ok
}
@@ -22,14 +22,17 @@ import (
"github.com/stretchr/testify/assert"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/utils/pointer"
"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
"github.com/oam-dev/kubevela/apis/interfaces"
"github.com/oam-dev/kubevela/pkg/oam"
)
func TestSetOAMOwner(t *testing.T) {
tests := map[string]struct {
OO ObjectOwner
OO interfaces.ObjectOwner
CO v1.OwnerReference
ExpOwner []v1.OwnerReference
}{
@@ -41,37 +44,11 @@ func TestSetOAMOwner(t *testing.T) {
Name: "myapp",
},
ExpOwner: []v1.OwnerReference{{
APIVersion: "core.oam.dev/v1beta1",
Kind: "ResourceTracker",
Name: "myapp",
}},
},
"test remove old resourceTracker owner": {
OO: &unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"ownerReferences": []interface{}{
map[string]interface{}{
"apiVersion": "core.oam.dev/v1beta1",
"kind": "ResourceTracker",
},
map[string]interface{}{
"apiVersion": "core.oam.dev/v1alpha2",
"kind": "ApplicationContext",
},
},
},
},
},
CO: v1.OwnerReference{
APIVersion: "core.oam.dev/v1beta1",
Kind: "ResourceTracker",
Name: "myapp",
},
ExpOwner: []v1.OwnerReference{{
APIVersion: "core.oam.dev/v1beta1",
Kind: "ResourceTracker",
Name: "myapp",
APIVersion: "core.oam.dev/v1beta1",
Kind: "ResourceTracker",
Name: "myapp",
Controller: pointer.Bool(true),
BlockOwnerDeletion: pointer.Bool(true),
}},
},
"test other owner not removed": {
@@ -94,19 +71,22 @@ func TestSetOAMOwner(t *testing.T) {
Name: "myapp",
},
ExpOwner: []v1.OwnerReference{{
APIVersion: "core.oam.dev/v1beta1",
Kind: "ResourceTracker",
Name: "myapp",
},
{
APIVersion: "core.oam.dev/v1alpha1",
Kind: "Rollout",
Name: "xxx",
}},
APIVersion: "core.oam.dev/v1beta1",
Kind: "ResourceTracker",
Name: "myapp",
Controller: pointer.Bool(true),
BlockOwnerDeletion: pointer.Bool(true),
}, {
APIVersion: "core.oam.dev/v1alpha1",
Kind: "Rollout",
Name: "xxx",
}},
},
}
for name, ti := range tests {
setOrOverrideOAMControllerOwner(ti.OO, ti.CO)
rt := &v1beta1.ResourceTracker{}
rt.Name = ti.CO.Name
rt.AddOwnerReferenceToTrackerResource(ti.OO)
assert.Equal(t, ti.ExpOwner, ti.OO.GetOwnerReferences(), name)
}
}
@@ -22,6 +22,7 @@ import (
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
)
@@ -33,31 +34,39 @@ func TestIsTrackedResources(t *testing.T) {
}{{
oldRT: &v1beta1.ResourceTracker{
Status: v1beta1.ResourceTrackerStatus{
TrackedResources: []corev1.ObjectReference{{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
TrackedResources: []common.ClusterObjectReference{{
ObjectReference: corev1.ObjectReference{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
},
}, {
Kind: "Pod",
APIVersion: "v1",
Name: "test",
Namespace: "default",
ObjectReference: corev1.ObjectReference{
Kind: "Pod",
APIVersion: "v1",
Name: "test",
Namespace: "default",
},
}},
},
},
newRT: &v1beta1.ResourceTracker{
Status: v1beta1.ResourceTrackerStatus{
TrackedResources: []corev1.ObjectReference{{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
TrackedResources: []common.ClusterObjectReference{{
ObjectReference: corev1.ObjectReference{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
},
}, {
Kind: "Pod",
APIVersion: "v1",
Name: "test",
Namespace: "default",
ObjectReference: corev1.ObjectReference{
Kind: "Pod",
APIVersion: "v1",
Name: "test",
Namespace: "default",
},
}},
},
},
@@ -65,31 +74,39 @@ func TestIsTrackedResources(t *testing.T) {
}, {
oldRT: &v1beta1.ResourceTracker{
Status: v1beta1.ResourceTrackerStatus{
TrackedResources: []corev1.ObjectReference{{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
TrackedResources: []common.ClusterObjectReference{{
ObjectReference: corev1.ObjectReference{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
},
}, {
Kind: "Pod",
APIVersion: "v1",
Name: "hello",
Namespace: "default",
ObjectReference: corev1.ObjectReference{
Kind: "Pod",
APIVersion: "v1",
Name: "hello",
Namespace: "default",
},
}},
},
},
newRT: &v1beta1.ResourceTracker{
Status: v1beta1.ResourceTrackerStatus{
TrackedResources: []corev1.ObjectReference{{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
TrackedResources: []common.ClusterObjectReference{{
ObjectReference: corev1.ObjectReference{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
},
}, {
Kind: "Pod",
APIVersion: "v1",
Name: "test",
Namespace: "default",
ObjectReference: corev1.ObjectReference{
Kind: "Pod",
APIVersion: "v1",
Name: "test",
Namespace: "default",
},
}},
},
},
@@ -98,16 +115,20 @@ func TestIsTrackedResources(t *testing.T) {
oldRT: &v1beta1.ResourceTracker{},
newRT: &v1beta1.ResourceTracker{
Status: v1beta1.ResourceTrackerStatus{
TrackedResources: []corev1.ObjectReference{{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
TrackedResources: []common.ClusterObjectReference{{
ObjectReference: corev1.ObjectReference{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
},
}, {
Kind: "Pod",
APIVersion: "v1",
Name: "test",
Namespace: "default",
ObjectReference: corev1.ObjectReference{
Kind: "Pod",
APIVersion: "v1",
Name: "test",
Namespace: "default",
},
}},
},
},
@@ -115,16 +136,20 @@ func TestIsTrackedResources(t *testing.T) {
}, {
oldRT: &v1beta1.ResourceTracker{
Status: v1beta1.ResourceTrackerStatus{
TrackedResources: []corev1.ObjectReference{{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
TrackedResources: []common.ClusterObjectReference{{
ObjectReference: corev1.ObjectReference{
Kind: "Deployment",
APIVersion: "apps/v1",
Name: "test",
Namespace: "default",
},
}, {
Kind: "Pod",
APIVersion: "v1",
Name: "test",
Namespace: "default",
ObjectReference: corev1.ObjectReference{
Kind: "Pod",
APIVersion: "v1",
Name: "test",
Namespace: "default",
},
}},
},
},
@@ -25,6 +25,7 @@ import (
"github.com/oam-dev/kubevela/pkg/cue/model"
"github.com/oam-dev/kubevela/pkg/multicluster"
"github.com/oam-dev/kubevela/pkg/resourcetracker"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
@@ -598,27 +599,6 @@ func ComputeComponentRevisionHash(comp *types.ComponentManifest) (string, error)
return utils.ComputeSpecHash(&compRevisionHash)
}
// createOrGetResourceTracker create or get a resource tracker to manage all componentRevisions
func (h *AppHandler) createOrGetResourceTracker(ctx context.Context) (*v1beta1.ResourceTracker, error) {
rt := &v1beta1.ResourceTracker{}
rtName := h.app.Name + "-" + h.app.Namespace
if err := h.r.Get(ctx, ktypes.NamespacedName{Name: rtName}, rt); err != nil {
if !apierrors.IsNotFound(err) {
return nil, err
}
rt.SetName(rtName)
rt.SetLabels(map[string]string{
oam.LabelAppName: h.app.Name,
oam.LabelAppNamespace: h.app.Namespace,
})
rt.SetAnnotations(map[string]string{oam.AnnotationResourceTrackerLifeLong: "true"})
if err = h.r.Create(ctx, rt); err != nil {
return nil, err
}
}
return rt, nil
}
// createControllerRevision records snapshot of a component
func (h *AppHandler) createControllerRevision(ctx context.Context, cm *types.ComponentManifest) error {
comp, err := componentManifest2Component(cm)
@@ -626,7 +606,7 @@ func (h *AppHandler) createControllerRevision(ctx context.Context, cm *types.Com
return err
}
revision, _ := utils.ExtractRevision(cm.RevisionName)
rt, err := h.createOrGetResourceTracker(ctx)
rt, err := resourcetracker.CreateOrGetApplicationRootResourceTracker(ctx, h.r.Client, h.app)
if err != nil {
return err
}
@@ -798,7 +778,7 @@ func gatherUsingAppRevision(ctx context.Context, h *AppHandler) (map[string]bool
return nil, err
}
for _, rt := range rtList.Items {
if dispatch.IsLifeLongResourceTracker(rt) {
if rt.IsLifeLong() {
continue
}
appRev := dispatch.ExtractAppRevisionName(rt.Name, ns)
@@ -25,7 +25,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
v1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -317,14 +316,7 @@ func (h *handler) recordWorkloadInResourceTracker(ctx context.Context, workload
klog.Errorf("fail to get resourceTracker to record workload rollout: namespace:%s, name: %s", h.rollout.Namespace, h.rollout.Name)
return err
}
recordedWorkload := corev1.ObjectReference{
APIVersion: workload.GetAPIVersion(),
Kind: workload.GetKind(),
UID: workload.GetUID(),
Namespace: workload.GetNamespace(),
Name: workload.GetName(),
}
rt.Status.TrackedResources = append(rt.Status.TrackedResources, recordedWorkload)
rt.AddTrackedResource(workload)
if err := h.Status().Update(ctx, &rt); err != nil {
klog.Errorf("fail to update resourceTracker for rollout record workload namespace:%s, name: %s", h.rollout.Namespace, h.rollout.Name)
return err