From b33b6fbead9cb89457e2ad2d904fc07e7b98c247 Mon Sep 17 00:00:00 2001 From: wyike <77846369+wangyikewxgm@users.noreply.github.com> Date: Wed, 7 Apr 2021 21:34:48 +0800 Subject: [PATCH] garbage collection for cross-namespace workloads and traits in application layer (#1421) * add logic for gc cross ns resources WIP regenerate api and charts rewrite handle resourceTracker logic WIP add e2e test logic add more e2e test * WIP refactor sevral funs name and add comments for code * WIP add more corener case e2e test * add unit test and fix bug * WIP refactor handleResouceTracker logic * refactor func name aglin with others and add comments * make generate to fix check-diff ci error * change resourceTracker status as a subresource --- apis/core.oam.dev/v1beta1/core_types.go | 30 ++ .../v1beta1/zz_generated.deepcopy.go | 36 ++ .../crds/core.oam.dev_resourcetrackers.yaml | 31 ++ docs/en/cue/cross-namespace-resource.md | 3 - .../crds/core.oam.dev_resourcetrackers.yaml | 31 ++ .../application/application_controller.go | 7 + .../application/application_finalizer_test.go | 122 +++-- .../v1alpha2/application/apply.go | 332 ++++++++---- test/e2e-test/app_resourcetracker_test.go | 489 +++++++++++++++++- 9 files changed, 947 insertions(+), 134 deletions(-) diff --git a/apis/core.oam.dev/v1beta1/core_types.go b/apis/core.oam.dev/v1beta1/core_types.go index b6140ad9b..abb7aabbf 100644 --- a/apis/core.oam.dev/v1beta1/core_types.go +++ b/apis/core.oam.dev/v1beta1/core_types.go @@ -20,6 +20,7 @@ import ( runtimev1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "github.com/oam-dev/kubevela/apis/core.oam.dev/common" ) @@ -236,12 +237,41 @@ type ScopeDefinitionList struct { } // +kubebuilder:object:root=true +// +kubebuilder:subresource:status // An ResourceTracker represents a tracker for track cross namespace resources // +kubebuilder:resource:scope=Cluster,categories={oam} type ResourceTracker struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` + + Status ResourceTrackerStatus `json:"status,omitempty"` +} + +// ResourceTrackerStatus define the status of resourceTracker +type ResourceTrackerStatus struct { + TrackedResources []TypedReference `json:"trackedResources,omitempty"` +} + +// A TypedReference refers to an object by Name, Kind, and APIVersion. It is +// commonly used to reference across-namespace objects +type TypedReference struct { + // APIVersion of the referenced object. + APIVersion string `json:"apiVersion"` + + // Kind of the referenced object. + Kind string `json:"kind"` + + // Name of the referenced object. + Name string `json:"name"` + + // Namespace of the objects outside the application namespace. + // +optional + Namespace string `json:"namespace,omitempty"` + + // UID of the referenced object. + // +optional + UID types.UID `json:"uid,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/core.oam.dev/v1beta1/zz_generated.deepcopy.go b/apis/core.oam.dev/v1beta1/zz_generated.deepcopy.go index 55eb26ed1..c8fd14b1e 100644 --- a/apis/core.oam.dev/v1beta1/zz_generated.deepcopy.go +++ b/apis/core.oam.dev/v1beta1/zz_generated.deepcopy.go @@ -862,6 +862,7 @@ func (in *ResourceTracker) DeepCopyInto(out *ResourceTracker) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTracker. @@ -914,6 +915,26 @@ func (in *ResourceTrackerList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceTrackerStatus) DeepCopyInto(out *ResourceTrackerStatus) { + *out = *in + if in.TrackedResources != nil { + in, out := &in.TrackedResources, &out.TrackedResources + *out = make([]TypedReference, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTrackerStatus. +func (in *ResourceTrackerStatus) DeepCopy() *ResourceTrackerStatus { + if in == nil { + return nil + } + out := new(ResourceTrackerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScopeDefinition) DeepCopyInto(out *ScopeDefinition) { *out = *in @@ -1141,6 +1162,21 @@ func (in *TraitDefinitionStatus) DeepCopy() *TraitDefinitionStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TypedReference) DeepCopyInto(out *TypedReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TypedReference. +func (in *TypedReference) DeepCopy() *TypedReference { + if in == nil { + return nil + } + out := new(TypedReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *URIMatch) DeepCopyInto(out *URIMatch) { *out = *in diff --git a/charts/vela-core/crds/core.oam.dev_resourcetrackers.yaml b/charts/vela-core/crds/core.oam.dev_resourcetrackers.yaml index 044049aa8..1141f6e5d 100644 --- a/charts/vela-core/crds/core.oam.dev_resourcetrackers.yaml +++ b/charts/vela-core/crds/core.oam.dev_resourcetrackers.yaml @@ -30,9 +30,40 @@ spec: type: string metadata: type: object + status: + description: ResourceTrackerStatus define the status of resourceTracker + properties: + trackedResources: + items: + description: A TypedReference refers to an object by Name, Kind, and APIVersion. It is commonly used to reference across-namespace objects + properties: + apiVersion: + description: APIVersion of the referenced object. + type: string + kind: + description: Kind of the referenced object. + type: string + name: + description: Name of the referenced object. + type: string + namespace: + description: Namespace of the objects outside the application namespace. + type: string + uid: + description: UID of the referenced object. + type: string + required: + - apiVersion + - kind + - name + type: object + type: array + type: object type: object served: true storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/docs/en/cue/cross-namespace-resource.md b/docs/en/cue/cross-namespace-resource.md index 533acd351..4fa19f3c5 100644 --- a/docs/en/cue/cross-namespace-resource.md +++ b/docs/en/cue/cross-namespace-resource.md @@ -50,7 +50,4 @@ spec: }}} } ``` -## Limitations -If you update definition by changing the `metadata.namespace` field. KubeVela will create new resources in the new namespace but not delete old resources. -We wil fix the limitation in the near future. diff --git a/legacy/charts/vela-core-legacy/crds/core.oam.dev_resourcetrackers.yaml b/legacy/charts/vela-core-legacy/crds/core.oam.dev_resourcetrackers.yaml index 8e986d091..d3816ecc6 100644 --- a/legacy/charts/vela-core-legacy/crds/core.oam.dev_resourcetrackers.yaml +++ b/legacy/charts/vela-core-legacy/crds/core.oam.dev_resourcetrackers.yaml @@ -16,6 +16,8 @@ spec: plural: resourcetrackers singular: resourcetracker scope: Cluster + subresources: + status: {} validation: openAPIV3Schema: description: An ResourceTracker represents a tracker for track cross namespace resources @@ -28,6 +30,35 @@ spec: type: string metadata: type: object + status: + description: ResourceTrackerStatus define the status of resourceTracker + properties: + trackedResources: + items: + description: A TypedReference refers to an object by Name, Kind, and APIVersion. It is commonly used to reference across-namespace objects + properties: + apiVersion: + description: APIVersion of the referenced object. + type: string + kind: + description: Kind of the referenced object. + type: string + name: + description: Name of the referenced object. + type: string + namespace: + description: Namespace of the objects outside the application namespace. + type: string + uid: + description: UID of the referenced object. + type: string + required: + - apiVersion + - kind + - name + type: object + type: array + type: object type: object version: v1beta1 versions: diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go index fdff2c6ac..a62bdf3d5 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go @@ -145,6 +145,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { err = handler.handleResourceTracker(ctx, comps, ac) if err != nil { applog.Error(err, "[Handle resourceTracker]") + app.Status.SetConditions(errorCondition("Handle resourceTracker", err)) return handler.handleErr(err) } @@ -180,6 +181,12 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { app.Status.Services = appCompStatus app.Status.SetConditions(readyCondition("HealthCheck")) app.Status.Phase = common.ApplicationRunning + err = handler.garbageCollection(ctx) + if err != nil { + applog.Error(err, "[Garbage collection]") + app.Status.SetConditions(errorCondition("GarbageCollection", err)) + return handler.handleErr(err) + } // Gather status of components var refComps []v1alpha1.TypedReference for _, comp := range comps { diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/application_finalizer_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/application_finalizer_test.go index f78683978..01acc976b 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/application_finalizer_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/application_finalizer_test.go @@ -166,6 +166,88 @@ var _ = Describe("Test application controller finalizer logic", func() { checkRt := new(v1beta1.ResourceTracker) Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), checkRt)).Should(util.NotFoundMatcher{}) }) + + It("Test cross namespace workload, then update the app to change the namespace", func() { + appName := "app-3" + appKey := types.NamespacedName{Namespace: namespace, Name: appName} + app := getApp(appName, namespace, "cross-worker") + Expect(k8sClient.Create(ctx, app)).Should(BeNil()) + + By("Create a cross workload app") + _, err := reconciler.Reconcile(ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + checkApp := &v1beta1.Application{} + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + Expect(checkApp.Status.Phase).Should(Equal(common.ApplicationRunning)) + Expect(len(checkApp.Finalizers)).Should(BeEquivalentTo(0)) + rt := &v1beta1.ResourceTracker{} + Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), rt)).Should(BeNil()) + _, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + checkApp = new(v1beta1.Application) + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + Expect(len(checkApp.Finalizers)).Should(BeEquivalentTo(1)) + Expect(checkApp.Finalizers[0]).Should(BeEquivalentTo(resourceTrackerFinalizer)) + Expect(checkApp.Status.ResourceTracker.UID).Should(BeEquivalentTo(rt.UID)) + Expect(len(rt.Status.TrackedResources)).Should(BeEquivalentTo(1)) + By("Update the app, set type to normal-worker") + checkApp.Spec.Components[0].Type = "normal-worker" + Expect(k8sClient.Update(ctx, checkApp)).Should(BeNil()) + _, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + checkApp = new(v1beta1.Application) + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + Expect(checkApp.Status.ResourceTracker).Should(BeNil()) + Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), rt)).Should(util.NotFoundMatcher{}) + Expect(k8sClient.Delete(ctx, checkApp)).Should(BeNil()) + _, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + }) + + It("Test cross namespace workload and trait, then update the app to delete trait ", func() { + appName := "app-4" + appKey := types.NamespacedName{Namespace: namespace, Name: appName} + app := getApp(appName, namespace, "cross-worker") + app.Spec.Components[0].Traits = []v1beta1.ApplicationTrait{ + { + Type: "cross-scaler", + Properties: runtime.RawExtension{Raw: []byte(`{"replicas": 1}`)}, + }, + } + Expect(k8sClient.Create(ctx, app)).Should(BeNil()) + By("Create a cross workload trait app") + _, err := reconciler.Reconcile(ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + checkApp := &v1beta1.Application{} + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + Expect(checkApp.Status.Phase).Should(Equal(common.ApplicationRunning)) + Expect(len(checkApp.Finalizers)).Should(BeEquivalentTo(0)) + rt := &v1beta1.ResourceTracker{} + Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), rt)).Should(BeNil()) + _, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + checkApp = new(v1beta1.Application) + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + Expect(len(checkApp.Finalizers)).Should(BeEquivalentTo(1)) + Expect(checkApp.Finalizers[0]).Should(BeEquivalentTo(resourceTrackerFinalizer)) + Expect(checkApp.Status.ResourceTracker.UID).Should(BeEquivalentTo(rt.UID)) + Expect(len(rt.Status.TrackedResources)).Should(BeEquivalentTo(2)) + By("Update the app, set type to normal-worker") + checkApp.Spec.Components[0].Traits = nil + Expect(k8sClient.Update(ctx, checkApp)).Should(BeNil()) + _, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + rt = &v1beta1.ResourceTracker{} + checkApp = new(v1beta1.Application) + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), rt)).Should(BeNil()) + Expect(checkApp.Status.ResourceTracker.UID).Should(BeEquivalentTo(rt.UID)) + Expect(len(rt.Status.TrackedResources)).Should(BeEquivalentTo(1)) + Expect(k8sClient.Delete(ctx, checkApp)).Should(BeNil()) + _, err = reconciler.Reconcile(ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + Expect(k8sClient.Get(ctx, getTrackerKey(checkApp.Namespace, checkApp.Name), rt)).Should(util.NotFoundMatcher{}) + }) }) var _ = Describe("Test finalizer related func", func() { @@ -186,46 +268,6 @@ var _ = Describe("Test finalizer related func", func() { By("[TEST] Clean up resources after an integration test") }) - It("Test getResourceTrackerAndOwnReference func", func() { - app := getApp("app-1", namespace, "worker") - handler = appHandler{ - r: reconciler, - app: app, - logger: reconciler.Log.WithValues("application", "finalizer-func-test"), - } - checkRt := new(v1beta1.ResourceTracker) - Expect(k8sClient.Get(ctx, getTrackerKey(namespace, app.Name), checkRt)).Should(util.NotFoundMatcher{}) - rt, owner, err := handler.getResourceTrackerAndOwnReference(ctx) - Expect(err).Should(BeNil()) - Expect(rt.UID).Should(BeEquivalentTo(owner.UID)) - Expect(owner.Kind).Should(BeEquivalentTo(v1beta1.ResourceTrackerKind)) - checkRt = new(v1beta1.ResourceTracker) - Expect(k8sClient.Get(ctx, getTrackerKey(namespace, app.Name), checkRt)).Should(BeNil()) - Expect(checkRt.UID).Should(BeEquivalentTo(rt.UID)) - Expect(k8sClient.Delete(ctx, checkRt)).Should(BeNil()) - }) - - It("Test getResourceTrackerAndOwnReference func with already exsit resourceTracker", func() { - app := getApp("app-2", namespace, "worker") - handler = appHandler{ - r: reconciler, - app: app, - logger: reconciler.Log.WithValues("application", "finalizer-func-test"), - } - rt := &v1beta1.ResourceTracker{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace + "-" + app.GetName(), - }, - } - Expect(k8sClient.Create(ctx, rt)).Should(BeNil()) - checkRt, owner, err := handler.getResourceTrackerAndOwnReference(ctx) - Expect(err).Should(BeNil()) - Expect(rt.UID).Should(BeEquivalentTo(checkRt.UID)) - Expect(owner.Kind).Should(BeEquivalentTo(v1beta1.ResourceTrackerKind)) - Expect(checkRt.UID).Should(BeEquivalentTo(owner.UID)) - Expect(k8sClient.Delete(ctx, checkRt)).Should(BeNil()) - }) - It("Test finalizeResourceTracker func with need update ", func() { app := getApp("app-3", namespace, "worker") rt := &v1beta1.ResourceTracker{ diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/apply.go b/pkg/controller/core.oam.dev/v1alpha2/application/apply.go index f6dba31b3..f1351137b 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/apply.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/apply.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strconv" + "strings" "time" runtimev1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1" @@ -31,6 +32,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" ctypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" @@ -41,6 +43,7 @@ import ( "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/pkg/appfile" + "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev/v1alpha2/applicationconfiguration" "github.com/oam-dev/kubevela/pkg/controller/utils" "github.com/oam-dev/kubevela/pkg/dsl/process" "github.com/oam-dev/kubevela/pkg/oam" @@ -67,13 +70,15 @@ func readyCondition(tpy string) runtimev1alpha1.Condition { } type appHandler struct { - r *Reconciler - app *v1beta1.Application - appfile *appfile.Appfile - logger logr.Logger - inplace bool - isNewRevision bool - revisionHash string + r *Reconciler + app *v1beta1.Application + appfile *appfile.Appfile + logger logr.Logger + inplace bool + isNewRevision bool + revisionHash string + acrossNamespaceResources []v1beta1.TypedReference + resourceTracker *v1beta1.ResourceTracker } // setInplace will mark if the application should upgrade the workload within the same instance(name never changed) @@ -108,6 +113,12 @@ func (h *appHandler) apply(ctx context.Context, appRev *v1beta1.ApplicationRevis Controller: pointer.BoolPtr(true), }} + if _, exist := h.app.GetAnnotations()[oam.AnnotationAppRollout]; !exist && h.app.Spec.RolloutPlan == nil { + h.setInplace(true) + } else { + h.setInplace(false) + } + // don't create components and AC if revision-only annotation is set if ac.Annotations[oam.AnnotationAppRevisionOnly] == "true" { h.FinalizeAppRevision(appRev, ac, comps) @@ -116,12 +127,21 @@ func (h *appHandler) apply(ctx context.Context, appRev *v1beta1.ApplicationRevis for _, comp := range comps { comp.SetOwnerReferences(owners) + needTracker, err := h.checkAndSetResourceTracker(&comp.Spec.Workload) + if err != nil { + return err + } newComp := comp.DeepCopy() // newComp will be updated and return the revision name instead of the component name revisionName, err := h.createOrUpdateComponent(ctx, newComp) if err != nil { return err } + if needTracker { + if err := h.recodeTrackedWorkload(comp, revisionName); err != nil { + return err + } + } // find the ACC that contains this component for i := 0; i < len(ac.Spec.Components); i++ { // update the AC using the component revision instead of component name @@ -129,6 +149,9 @@ func (h *appHandler) apply(ctx context.Context, appRev *v1beta1.ApplicationRevis if ac.Spec.Components[i].ComponentName == newComp.Name { ac.Spec.Components[i].RevisionName = revisionName ac.Spec.Components[i].ComponentName = "" + if err := h.checkResourceTrackerForTrait(ctx, ac.Spec.Components[i], newComp.Name); err != nil { + return err + } } } if comp.Spec.Helm != nil { @@ -148,11 +171,9 @@ func (h *appHandler) apply(ctx context.Context, appRev *v1beta1.ApplicationRevis // the rollout will create AppContext which will launch the real K8s resources. // Otherwise, we should create/update the appContext here when there if no rollout controller to take care of new versions // In this case, the workload should update with the annotation `app.oam.dev/inplace-upgrade=true` - if _, exist := h.app.GetAnnotations()[oam.AnnotationAppRollout]; !exist && h.app.Spec.RolloutPlan == nil { - h.setInplace(true) + if h.inplace { return h.createOrUpdateAppContext(ctx, owners) } - h.setInplace(false) return nil } @@ -399,90 +420,30 @@ func (h *appHandler) applyHelmModuleResources(ctx context.Context, comp *v1alpha return nil } -// handleResourceTracker check the namesapce of all components and traits, if the namespace is different with application, the tracker will own them -func (h *appHandler) handleResourceTracker(ctx context.Context, components []*v1alpha2.Component, ac *v1alpha2.ApplicationConfiguration) error { - ref := new(metav1.OwnerReference) - // resourceTracker is cache for resourceTracker, avoid get from k8s every time - resourceTracker := new(v1beta1.ResourceTracker) +// checkAndSetResourceTracker check if resource's namespace is different with application, if yes set resourceTracker as +// resource's ownerReference +func (h *appHandler) checkAndSetResourceTracker(resource *runtime.RawExtension) (bool, error) { needTracker := false - for i, c := range components { - u, err := oamutil.RawExtension2Unstructured(&c.Spec.Workload) - if err != nil { - return err - } - if checkResourceDiffWithApp(u, h.app.Namespace) { - needTracker = true - if len(resourceTracker.Name) == 0 { - resourceTracker, ref, err = h.getResourceTrackerAndOwnReference(ctx) - if err != nil { - return err - } - } - u.SetOwnerReferences([]metav1.OwnerReference{*ref}) - raw := oamutil.Object2RawExtension(u) - components[i].Spec.Workload = raw - } + u, err := oamutil.RawExtension2Unstructured(resource) + if err != nil { + return false, err } - for _, acComponent := range ac.Spec.Components { - for i, t := range acComponent.Traits { - u, err := oamutil.RawExtension2Unstructured(&t.Trait) - if err != nil { - return err - } - if checkResourceDiffWithApp(u, h.app.Namespace) { - needTracker = true - if len(resourceTracker.Name) == 0 { - resourceTracker, ref, err = h.getResourceTrackerAndOwnReference(ctx) - if err != nil { - return err - } - } - u.SetOwnerReferences([]metav1.OwnerReference{*ref}) - raw := oamutil.Object2RawExtension(u) - acComponent.Traits[i].Trait = raw - } - } + if checkResourceDiffWithApp(u, h.app.Namespace) { + needTracker = true + ref := h.genResourceTrackerOwnerReference() + // set resourceTracker as the ownerReference of workload/trait + u.SetOwnerReferences([]metav1.OwnerReference{*ref}) + raw := oamutil.Object2RawExtension(u) + *resource = raw + return needTracker, nil } - if !needTracker { - h.app.Status.ResourceTracker = nil - // check weather related resourceTracker is existed, if yes delete it - err := h.r.Get(ctx, ctypes.NamespacedName{Name: h.generateResourceTrackerName()}, resourceTracker) - if err == nil { - return h.r.Delete(ctx, resourceTracker) - } - if !apierrors.IsNotFound(err) { - return err - } - return nil - } - h.app.Status.ResourceTracker = &runtimev1alpha1.TypedReference{ - Name: resourceTracker.Name, - Kind: v1beta1.ResourceTrackerGroupKind, - APIVersion: v1beta1.ResourceTrackerKindAPIVersion, - UID: resourceTracker.UID} - - return nil + return needTracker, nil } -func (h *appHandler) getResourceTrackerAndOwnReference(ctx context.Context) (*v1beta1.ResourceTracker, *metav1.OwnerReference, error) { - resourceTracker := new(v1beta1.ResourceTracker) - key := ctypes.NamespacedName{Name: h.generateResourceTrackerName()} - err := h.r.Get(ctx, key, resourceTracker) - if err != nil { - if apierrors.IsNotFound(err) { - resourceTracker = &v1beta1.ResourceTracker{ - ObjectMeta: metav1.ObjectMeta{ - Name: h.generateResourceTrackerName(), - }, - } - if err = h.r.Client.Create(ctx, resourceTracker); err != nil { - return nil, nil, err - } - return resourceTracker, metav1.NewControllerRef(resourceTracker, v1beta1.ResourceTrackerKindVersionKind), nil - } - return nil, nil, err - } - return resourceTracker, metav1.NewControllerRef(resourceTracker, v1beta1.ResourceTrackerKindVersionKind), nil +// genResourceTrackerOwnerReference check the related resourceTracker whether have been created. +// If not, create it. And return the ownerReference of this resourceTracker. +func (h *appHandler) genResourceTrackerOwnerReference() *metav1.OwnerReference { + return metav1.NewControllerRef(h.resourceTracker, v1beta1.ResourceTrackerKindVersionKind) } func (h *appHandler) generateResourceTrackerName() string { @@ -529,3 +490,196 @@ func (h *appHandler) removeResourceTracker(ctx context.Context) (bool, error) { h.app.Status.ResourceTracker = nil return true, nil } + +func (h *appHandler) recodeTrackedWorkload(comp *v1alpha2.Component, compRevisionName string) error { + workloadName, err := h.getWorkloadName(comp.Spec.Workload, comp.Name, compRevisionName) + if err != nil { + return err + } + if err = h.recodeTrackedResource(workloadName, comp.Spec.Workload); err != nil { + return err + } + return nil +} + +// checkResourceTrackerForTrait check component trait namespace, if it's namespace is different with application, set resourceTracker as its ownerReference +// and recode trait in handler acrossNamespace field +func (h *appHandler) checkResourceTrackerForTrait(ctx context.Context, comp v1alpha2.ApplicationConfigurationComponent, compName string) error { + for i, ct := range comp.Traits { + needTracker, err := h.checkAndSetResourceTracker(&comp.Traits[i].Trait) + if err != nil { + return err + } + if needTracker { + traitName, err := h.getTraitName(ctx, compName, comp.Traits[i].DeepCopy(), &ct.Trait) + if err != nil { + return err + } + if err = h.recodeTrackedResource(traitName, ct.Trait); err != nil { + return err + } + } + } + return nil +} + +// getWorkloadName generate workload name. By default the workload's name will be generated by applicationContext, this func is for application controller +// get name of crossNamespace workload. The logic of this func is same with the way of appConfig generating workloadName +func (h *appHandler) getWorkloadName(w runtime.RawExtension, componentName string, revisionName string) (string, error) { + workload, err := oamutil.RawExtension2Unstructured(&w) + if err != nil { + return "", err + } + var revision int = 0 + if len(revisionName) != 0 { + r, err := utils.ExtractRevision(revisionName) + if err != nil { + return "", err + } + revision = r + } + applicationconfiguration.SetAppWorkloadInstanceName(componentName, workload, revision, strconv.FormatBool(h.inplace)) + return workload.GetName(), nil +} + +// getTraitName generate trait name. By default the trait name will be generated by applicationContext, this func is for application controller +// get name of crossNamespace trait. The logic of this func is same with the way of appConfig generating traitName +func (h *appHandler) getTraitName(ctx context.Context, componentName string, ct *v1alpha2.ComponentTrait, t *runtime.RawExtension) (string, error) { + trait, err := oamutil.RawExtension2Unstructured(t) + if err != nil { + return "", err + } + traitDef, err := oamutil.FetchTraitDefinition(ctx, h.r, h.r.dm, trait) + if err != nil { + if !apierrors.IsNotFound(err) { + return "", errors.Wrapf(err, "cannot find trait definition %q %q %q", trait.GetAPIVersion(), trait.GetKind(), trait.GetName()) + } + traitDef = oamutil.GetDummyTraitDefinition(trait) + } + traitType := traitDef.Name + if strings.Contains(traitType, ".") { + traitType = strings.Split(traitType, ".")[0] + } + traitName := oamutil.GenTraitName(componentName, ct, traitType) + return traitName, nil +} + +// recodeTrackedResource append cross namespace resource to apphandler's acrossNamespaceResources field +func (h *appHandler) recodeTrackedResource(resourceName string, resource runtime.RawExtension) error { + u, err := oamutil.RawExtension2Unstructured(&resource) + if err != nil { + return err + } + tr := new(v1beta1.TypedReference) + tr.Name = resourceName + tr.Namespace = u.GetNamespace() + tr.APIVersion = u.GetAPIVersion() + tr.Kind = u.GetKind() + h.acrossNamespaceResources = append(h.acrossNamespaceResources, *tr) + return nil +} + +// Now if workloads or traits are in the same namespace with application, applicationContext will take over gc workloads and traits. +// Here we cover the case in witch a cross namespace component or one of its cross namespace trait is removed from an application. +func (h *appHandler) garbageCollection(ctx context.Context) error { + rt := new(v1beta1.ResourceTracker) + err := h.r.Get(ctx, ctypes.NamespacedName{Name: h.generateResourceTrackerName()}, rt) + if err != nil { + if apierrors.IsNotFound(err) { + // guarantee app status right + h.app.Status.ResourceTracker = nil + return nil + } + return err + } + applied := map[v1beta1.TypedReference]bool{} + if len(h.acrossNamespaceResources) == 0 { + h.app.Status.ResourceTracker = nil + if err := h.r.Delete(ctx, rt); err != nil { + return err + } + return nil + } + for _, resource := range h.acrossNamespaceResources { + applied[resource] = true + } + for _, ref := range rt.Status.TrackedResources { + if !applied[ref] { + resource := new(unstructured.Unstructured) + resource.SetAPIVersion(ref.APIVersion) + resource.SetKind(ref.Kind) + resource.SetNamespace(ref.Namespace) + resource.SetName(ref.Name) + err := h.r.Delete(ctx, resource) + if err != nil { + if apierrors.IsNotFound(err) { + continue + } + return err + } + } + } + // update resourceTracker status, recode applied across-namespace resources + rt.Status.TrackedResources = h.acrossNamespaceResources + if err := h.r.Status().Update(ctx, rt); err != nil { + return err + } + h.app.Status.ResourceTracker = &runtimev1alpha1.TypedReference{ + Name: rt.Name, + Kind: v1beta1.ResourceTrackerGroupKind, + APIVersion: v1beta1.ResourceTrackerKindAPIVersion, + UID: rt.UID} + return nil +} + +// handleResourceTracker check the namespace of all workloads and traits +// if one resource is across-namespace create resourceTracker and set in appHandler field +func (h *appHandler) handleResourceTracker(ctx context.Context, components []*v1alpha2.Component, ac *v1alpha2.ApplicationConfiguration) error { + resourceTracker := new(v1beta1.ResourceTracker) + needTracker := false + for _, c := range components { + u, err := oamutil.RawExtension2Unstructured(&c.Spec.Workload) + if err != nil { + return err + } + if checkResourceDiffWithApp(u, h.app.Namespace) { + needTracker = true + break + } + } +outLoop: + for _, acComponent := range ac.Spec.Components { + for _, t := range acComponent.Traits { + u, err := oamutil.RawExtension2Unstructured(&t.Trait) + if err != nil { + return err + } + if checkResourceDiffWithApp(u, h.app.Namespace) { + needTracker = true + break outLoop + } + } + } + if needTracker { + // check weather related resourceTracker is existed, if not create it + err := h.r.Get(ctx, ctypes.NamespacedName{Name: h.generateResourceTrackerName()}, resourceTracker) + if err == nil { + h.resourceTracker = resourceTracker + return nil + } + if apierrors.IsNotFound(err) { + resourceTracker = &v1beta1.ResourceTracker{ + ObjectMeta: metav1.ObjectMeta{ + Name: h.generateResourceTrackerName(), + }, + } + if err = h.r.Client.Create(ctx, resourceTracker); err != nil { + return err + } + h.resourceTracker = resourceTracker + return nil + } + return err + } + return nil +} diff --git a/test/e2e-test/app_resourcetracker_test.go b/test/e2e-test/app_resourcetracker_test.go index 08fd31046..c1bab0ce4 100644 --- a/test/e2e-test/app_resourcetracker_test.go +++ b/test/e2e-test/app_resourcetracker_test.go @@ -127,7 +127,7 @@ var _ = Describe("Test application cross namespace resource", func() { return fmt.Errorf("appication status error ") } return nil - }, time.Second*600, time.Microsecond*300).Should(BeNil()) + }, time.Second*300, time.Microsecond*300).Should(BeNil()) By("check resource is generated correctly") Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil()) var workload appsv1.Deployment @@ -136,6 +136,10 @@ var _ = Describe("Test application cross namespace resource", func() { if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, appContext); err != nil { return fmt.Errorf("cannot generate AppContext %v", err) } + checkRt := new(v1beta1.ResourceTracker) + if err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), checkRt); err != nil { + return err + } component := &v1alpha2.Component{} if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: componentName}, component); err != nil { return fmt.Errorf("cannot generate component %v", err) @@ -158,6 +162,12 @@ var _ = Describe("Test application cross namespace resource", func() { if len(workload.OwnerReferences) != 1 || workload.OwnerReferences[0].UID != resourceTracker.UID { return fmt.Errorf("wrokload ownerreference error") } + if len(checkRt.Status.TrackedResources) != 1 { + return fmt.Errorf("resourceTracker status recode trackedResource length missmatch") + } + if checkRt.Status.TrackedResources[0].Name != workload.Name { + return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", checkRt.Status.TrackedResources[0].Name, workload.Name) + } return nil }, time.Second*50, time.Microsecond*300).Should(BeNil()) @@ -296,6 +306,12 @@ var _ = Describe("Test application cross namespace resource", func() { if len(trait.OwnerReferences) != 1 || trait.OwnerReferences[0].UID != resourceTracker.UID { return fmt.Errorf("trait owner reference missmatch") } + if len(resourceTracker.Status.TrackedResources) != 1 { + return fmt.Errorf("resourceTracker status recode trackedResource length missmatch") + } + if resourceTracker.Status.TrackedResources[0].Name != trait.Name { + return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, trait.Name) + } return nil }, time.Second*60, time.Microsecond*300).Should(BeNil()) }) @@ -370,6 +386,12 @@ var _ = Describe("Test application cross namespace resource", func() { if len(trait.OwnerReferences) != 1 || trait.OwnerReferences[0].UID != resourceTracker.UID { return fmt.Errorf("trait owner reference missmatch") } + if len(resourceTracker.Status.TrackedResources) != 1 { + return fmt.Errorf("resourceTracker status recode trackedResource length missmatch") + } + if resourceTracker.Status.TrackedResources[0].Name != trait.Name { + return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, trait.Name) + } return nil }, time.Second*60, time.Microsecond*300).Should(BeNil()) @@ -500,6 +522,12 @@ var _ = Describe("Test application cross namespace resource", func() { if app.Status.ResourceTracker == nil || app.Status.ResourceTracker.UID != resourceTracker.UID { return fmt.Errorf("app status resourceTracker error") } + if len(resourceTracker.Status.TrackedResources) != 1 { + return fmt.Errorf("resourceTracker status recode trackedResource length missmatch") + } + if resourceTracker.Status.TrackedResources[0].Name != crossDeplpoy.Name { + return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, crossDeplpoy.Name) + } return nil }, time.Second*60, time.Microsecond*300).Should(BeNil()) By("update application by delete cross namespace workload, resource tracker will be deleted, then check app status") @@ -547,7 +575,7 @@ var _ = Describe("Test application cross namespace resource", func() { return fmt.Errorf("error : cross namespace workload still exist") } if app.Status.ResourceTracker != nil { - return fmt.Errorf("errror app status resourceTracker") + return fmt.Errorf("error app status resourceTracker") } return nil }, time.Second*60, time.Microsecond*300).Should(BeNil()) @@ -625,12 +653,21 @@ var _ = Describe("Test application cross namespace resource", func() { return fmt.Errorf("error workload number %v", err) } workload = depolys.Items[0] + if err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker); err != nil { + return err + } if len(workload.OwnerReferences) != 1 || workload.OwnerReferences[0].UID != resourceTracker.UID { return fmt.Errorf("wrokload ownerreference error") } if workload.Spec.Template.Spec.Containers[0].Image != "busybox" { return fmt.Errorf("container image not match") } + if len(resourceTracker.Status.TrackedResources) != 1 { + return fmt.Errorf("resourceTracker status recode trackedResource length missmatch") + } + if resourceTracker.Status.TrackedResources[0].Name != workload.Name { + return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, workload.Name) + } return nil }, time.Second*50, time.Microsecond*300).Should(BeNil()) @@ -679,6 +716,15 @@ var _ = Describe("Test application cross namespace resource", func() { if workload.Spec.Template.Spec.Containers[0].Image != "nginx" { return fmt.Errorf("container image not match") } + if err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker); err != nil { + return err + } + if len(resourceTracker.Status.TrackedResources) != 1 { + return fmt.Errorf("resourceTracker status recode trackedResource length missmatch") + } + if resourceTracker.Status.TrackedResources[0].Name != workload.Name { + return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, workload.Name) + } return nil }, time.Second*60, time.Microsecond*1000).Should(BeNil()) @@ -704,6 +750,445 @@ var _ = Describe("Test application cross namespace resource", func() { return nil }, time.Second*30, time.Microsecond*300).Should(BeNil()) }) + + It("Test cross-namespace resource gc logic, delete a cross-ns component", func() { + var ( + appName = "test-app-6" + app = new(v1beta1.Application) + component1Name = "test-app-6-comp-1" + component2Name = "test-app-6-comp-2" + ) + By("install related definition") + + crossCdJson, err := yaml.YAMLToJSON([]byte(crossCompDefYaml)) + Expect(err).Should(BeNil()) + ctd := new(v1beta1.ComponentDefinition) + Expect(json.Unmarshal(crossCdJson, ctd)).Should(BeNil()) + Expect(k8sClient.Create(ctx, ctd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + + app = &v1beta1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: appName, + Namespace: namespace, + }, + Spec: v1beta1.ApplicationSpec{ + Components: []v1beta1.ApplicationComponent{ + v1beta1.ApplicationComponent{ + Name: component1Name, + Type: "cross-worker", + Properties: runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)}, + }, + v1beta1.ApplicationComponent{ + Name: component2Name, + Type: "cross-worker", + Properties: runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)}, + }, + }, + }, + } + + Expect(k8sClient.Create(ctx, app)).Should(BeNil()) + time.Sleep(3 * time.Second) // give informer cache to sync + resourceTracker := new(v1beta1.ResourceTracker) + + By("create application will generate two workload, and generate resourceTracker") + Eventually(func() error { + app = new(v1beta1.Application) + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app); err != nil { + return fmt.Errorf("error to get application %v", err) + } + if app.Status.Phase != common.ApplicationRunning { + return fmt.Errorf("application status not running") + } + err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker) + if err != nil { + return fmt.Errorf("error to generate resourceTracker %v", err) + } + crossOpts := []client.ListOption{ + client.InNamespace(crossNamespace), + client.MatchingLabels{ + oam.LabelAppName: appName, + }, + } + //same, cross := new(appsv1.DeploymentList), new(appsv1.DeploymentList) + workloads := new(appsv1.DeploymentList) + err = k8sClient.List(ctx, workloads, crossOpts...) + if err != nil || len(workloads.Items) != 2 { + return fmt.Errorf("failed get workloads") + } + deploy1 := workloads.Items[0] + if len(deploy1.OwnerReferences) != 1 || deploy1.OwnerReferences[0].Kind != v1beta1.ResourceTrackerKind { + return fmt.Errorf("deploy1 have error ownerReference") + } + deploy2 := workloads.Items[1] + if len(deploy2.OwnerReferences) != 1 || deploy2.OwnerReferences[0].UID != resourceTracker.UID { + return fmt.Errorf("deploy2 have error ownerReference") + } + if app.Status.ResourceTracker == nil || app.Status.ResourceTracker.UID != resourceTracker.UID { + return fmt.Errorf("app status resourceTracker error") + } + if len(resourceTracker.Status.TrackedResources) != 2 { + return fmt.Errorf("resourceTracker status recode trackedResource length missmatch") + } + if resourceTracker.Status.TrackedResources[0].Namespace != crossNamespace || resourceTracker.Status.TrackedResources[1].Namespace != crossNamespace { + return fmt.Errorf("resourceTracker recorde namespace mismatch") + } + if resourceTracker.Status.TrackedResources[0].Name != deploy1.Name && resourceTracker.Status.TrackedResources[1].Name != deploy1.Name { + return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, deploy1.Name) + } + if resourceTracker.Status.TrackedResources[0].Name != deploy2.Name && resourceTracker.Status.TrackedResources[1].Name != deploy2.Name { + return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", resourceTracker.Status.TrackedResources[0].Name, deploy2.Name) + } + return nil + }, time.Second*60, time.Microsecond*300).Should(BeNil()) + By("update application by delete a cross namespace workload, resource tracker will still exist, then check app status") + Eventually(func() error { + app = new(v1beta1.Application) + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil()) + app.Spec.Components = app.Spec.Components[:1] // delete a component + return k8sClient.Update(ctx, app) + }, time.Second*30, time.Microsecond*300).Should(BeNil()) + Eventually(func() error { + app = new(v1beta1.Application) + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app); err != nil { + return fmt.Errorf("error to get application %v", err) + } + if app.Status.Phase != common.ApplicationRunning { + return fmt.Errorf("application status not running") + } + err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker) + if err != nil { + return fmt.Errorf("failed to get resourceTracker %v", err) + } + crossOpts := []client.ListOption{ + client.InNamespace(crossNamespace), + client.MatchingLabels{ + oam.LabelAppName: appName, + }, + } + workloads := new(appsv1.DeploymentList) + err = k8sClient.List(ctx, workloads, crossOpts...) + if err != nil || len(workloads.Items) != 1 { + return fmt.Errorf("failed get cross namespace workload") + } + deploy := workloads.Items[0] + if len(deploy.OwnerReferences) != 1 || deploy.OwnerReferences[0].Kind != v1beta1.ResourceTrackerKind { + return fmt.Errorf("same ns deploy have error ownerReference") + } + checkRt := new(v1beta1.ResourceTracker) + err = k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), checkRt) + if err != nil { + return fmt.Errorf("error get resourceTracker") + } + if app.Status.ResourceTracker == nil { + return fmt.Errorf("app status resourceTracker error") + } + if app.Status.ResourceTracker.UID != checkRt.UID { + return fmt.Errorf("error app status resourceTracker UID") + } + if len(checkRt.Status.TrackedResources) != 1 { + return fmt.Errorf("error resourceTracker status trackedResource") + } + return nil + }, time.Second*80, time.Microsecond*300).Should(BeNil()) + + By("deleting application will remove resourceTracker and related resourceTracker will be removed") + app = new(v1beta1.Application) + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil()) + Expect(k8sClient.Delete(ctx, app)).Should(BeNil()) + Eventually(func() error { + err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker) + if err == nil { + return fmt.Errorf("resourceTracker still exist") + } + if !apierrors.IsNotFound(err) { + return err + } + return nil + }, time.Second*60, time.Microsecond*300).Should(BeNil()) + }) + + It("Test cross-namespace resource gc logic, delete a cross-ns trait", func() { + var ( + appName = "test-app-7" + app = new(v1beta1.Application) + componentName = "test-app-7-comp" + ) + By("install related definition") + + crossCdJson, err := yaml.YAMLToJSON([]byte(crossCompDefYaml)) + Expect(err).Should(BeNil()) + ctd := new(v1beta1.ComponentDefinition) + Expect(json.Unmarshal(crossCdJson, ctd)).Should(BeNil()) + Expect(k8sClient.Create(ctx, ctd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + + crossTdJson, err := yaml.YAMLToJSON([]byte(crossNsTdYaml)) + Expect(err).Should(BeNil()) + td := new(v1beta1.TraitDefinition) + Expect(json.Unmarshal(crossTdJson, td)).Should(BeNil()) + Expect(k8sClient.Create(ctx, td)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + + app = &v1beta1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: appName, + Namespace: namespace, + }, + Spec: v1beta1.ApplicationSpec{ + Components: []v1beta1.ApplicationComponent{ + v1beta1.ApplicationComponent{ + Name: componentName, + Type: "cross-worker", + Properties: runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)}, + Traits: []v1beta1.ApplicationTrait{ + v1beta1.ApplicationTrait{ + Type: "cross-scaler", + Properties: runtime.RawExtension{Raw: []byte(`{"replicas": 0}`)}, + }, + }, + }, + }, + }, + } + + Expect(k8sClient.Create(ctx, app)).Should(BeNil()) + time.Sleep(3 * time.Second) // give informer cache to sync + resourceTracker := new(v1beta1.ResourceTracker) + By("create app and check resource and app status") + Eventually(func() error { + app = new(v1beta1.Application) + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app); err != nil { + return fmt.Errorf("error to get application %v", err) + } + if app.Status.Phase != common.ApplicationRunning { + return fmt.Errorf("application status not running") + } + err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker) + if err != nil { + return fmt.Errorf("error to get resourceTracker %v", err) + } + mts := new(v1alpha2.ManualScalerTraitList) + opts := []client.ListOption{ + client.InNamespace(crossNamespace), + client.MatchingLabels{ + oam.LabelAppName: appName, + }, + } + err = k8sClient.List(ctx, mts, opts...) + if err != nil || len(mts.Items) != 1 { + return fmt.Errorf("failed generate cross namespace trait") + } + if len(resourceTracker.Status.TrackedResources) != 2 { + return fmt.Errorf("resourceTracker status recode trackedResource length missmatch") + } + trait := mts.Items[0] + if len(trait.OwnerReferences) != 1 || trait.OwnerReferences[0].UID != resourceTracker.UID { + return fmt.Errorf("trait owner reference missmatch") + } + deploys := new(appsv1.DeploymentList) + err = k8sClient.List(ctx, deploys, opts...) + if err != nil || len(deploys.Items) != 1 { + return fmt.Errorf("error to list deploy") + } + deploy := deploys.Items[0] + if deploy.OwnerReferences[0].UID != resourceTracker.UID { + return fmt.Errorf("deploy owner reference missmatch") + } + for _, resource := range resourceTracker.Status.TrackedResources { + if resource.Kind == deploy.Kind && resource.Name != deploy.Name { + return fmt.Errorf("deploy name mismatch ") + } + if resource.Kind == trait.Kind && resource.Name != trait.Name { + return fmt.Errorf("trait name mismatch") + } + } + return nil + }, time.Second*60, time.Microsecond*300).Should(BeNil()) + + By("update application trait by delete cross ns trait, resourceTracker will still exist") + Eventually(func() error { + app = new(v1beta1.Application) + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil()) + app.Spec.Components[0].Traits = []v1beta1.ApplicationTrait{} + return k8sClient.Update(ctx, app) + }, time.Second*30, time.Microsecond*300).Should(BeNil()) + + Eventually(func() error { + app = new(v1beta1.Application) + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app); err != nil { + return fmt.Errorf("error to get application %v", err) + } + if app.Status.Phase != common.ApplicationRunning { + return fmt.Errorf("application status not running") + } + err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker) + if err != nil { + return fmt.Errorf("error to get resourceTracker %v", err) + } + mts := new(v1alpha2.ManualScalerTraitList) + opts := []client.ListOption{ + client.InNamespace(crossNamespace), + client.MatchingLabels{ + oam.LabelAppName: appName, + }, + } + err = k8sClient.List(ctx, mts, opts...) + if err != nil || len(mts.Items) != 0 { + return fmt.Errorf("cross namespace trait still exist") + } + if len(resourceTracker.Status.TrackedResources) != 1 { + return fmt.Errorf("resourceTracker status recode trackedResource length missmatch") + } + deploys := new(appsv1.DeploymentList) + err = k8sClient.List(ctx, deploys, opts...) + if err != nil || len(deploys.Items) != 1 { + return fmt.Errorf("error to list deploy") + } + deploy := deploys.Items[0] + if len(deploy.OwnerReferences) != 1 || deploy.OwnerReferences[0].UID != resourceTracker.UID { + return fmt.Errorf("deploy owner reference missmatch") + } + if resourceTracker.Status.TrackedResources[0].Name != deploy.Name { + return fmt.Errorf("error to record deploy name in app status") + } + return nil + }, time.Second*60, time.Microsecond*300).Should(BeNil()) + By("deleting application will remove resourceTracker and related resourceTracker will be removed") + app = new(v1beta1.Application) + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil()) + Expect(k8sClient.Delete(ctx, app)).Should(BeNil()) + Eventually(func() error { + err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker) + if err == nil { + return fmt.Errorf("resourceTracker still exist") + } + if !apierrors.IsNotFound(err) { + return err + } + return nil + }, time.Second*60, time.Microsecond*300).Should(BeNil()) + }) + + It("Test cross-namespace resource gc logic, update a cross-ns workload's namespace", func() { + // install related definition + crossCdJson, _ := yaml.YAMLToJSON([]byte(crossCompDefYaml)) + ccd := new(v1beta1.ComponentDefinition) + Expect(json.Unmarshal(crossCdJson, ccd)).Should(BeNil()) + Expect(k8sClient.Create(ctx, ccd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + + normalCdJson, _ := yaml.YAMLToJSON([]byte(normalCompDefYaml)) + ncd := new(v1beta1.ComponentDefinition) + Expect(json.Unmarshal(normalCdJson, ncd)).Should(BeNil()) + Expect(k8sClient.Create(ctx, ncd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + + var ( + appName = "test-app-8" + app = new(v1beta1.Application) + componentName = "test-app-8-comp" + ) + app = &v1beta1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: appName, + Namespace: namespace, + }, + Spec: v1beta1.ApplicationSpec{ + Components: []v1beta1.ApplicationComponent{ + v1beta1.ApplicationComponent{ + Name: componentName, + Type: "cross-worker", + Properties: runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)}, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, app)).Should(BeNil()) + By("check resource tracker has been created and app status ") + resourceTracker := new(v1beta1.ResourceTracker) + Eventually(func() error { + app := new(v1beta1.Application) + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app); err != nil { + return fmt.Errorf("app not found %v", err) + } + if err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker); err != nil { + return err + } + if app.Status.Phase != common.ApplicationRunning { + return fmt.Errorf("application status is not running") + } + if app.Status.ResourceTracker == nil || app.Status.ResourceTracker.UID != resourceTracker.UID { + return fmt.Errorf("appication status error ") + } + return nil + }, time.Second*60, time.Microsecond*300).Should(BeNil()) + By("check resource is generated correctly") + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app)).Should(BeNil()) + var workload appsv1.Deployment + Eventually(func() error { + checkRt := new(v1beta1.ResourceTracker) + if err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), checkRt); err != nil { + return err + } + depolys := new(appsv1.DeploymentList) + opts := []client.ListOption{ + client.InNamespace(crossNamespace), + client.MatchingLabels{ + oam.LabelAppName: appName, + }, + } + err := k8sClient.List(ctx, depolys, opts...) + if err != nil || len(depolys.Items) != 1 { + return fmt.Errorf("error workload number %v", err) + } + workload = depolys.Items[0] + if len(workload.OwnerReferences) != 1 || workload.OwnerReferences[0].UID != resourceTracker.UID { + return fmt.Errorf("wrokload ownerreference error") + } + if len(checkRt.Status.TrackedResources) != 1 { + return fmt.Errorf("resourceTracker status recode trackedResource length missmatch") + } + if checkRt.Status.TrackedResources[0].Name != workload.Name { + return fmt.Errorf("resourceTracker status recode trackedResource name mismatch recorded %s, actually %s", checkRt.Status.TrackedResources[0].Name, workload.Name) + } + return nil + }, time.Second*60, time.Microsecond*300).Should(BeNil()) + + By("update application modify workload namespace will remove resourceTracker and related old workload will be removed") + time.Sleep(3 * time.Second) // wait informer cache to be synced + Eventually(func() error { + app = new(v1beta1.Application) + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: appName}, app) + if err != nil { + return err + } + app.Spec.Components[0].Type = "normal-worker" + err = k8sClient.Update(ctx, app) + if err != nil { + return err + } + return nil + }, time.Second*30, time.Microsecond).Should(BeNil()) + Eventually(func() error { + err := k8sClient.Get(ctx, generateResourceTrackerKey(app.Namespace, app.Name), resourceTracker) + if err == nil { + return fmt.Errorf("resourceTracker still exist") + } + if !apierrors.IsNotFound(err) { + return err + } + err = k8sClient.Get(ctx, types.NamespacedName{Namespace: crossNamespace, Name: workload.GetName()}, &workload) + if err == nil { + return fmt.Errorf("wrokload still exist") + } + if !apierrors.IsNotFound(err) { + return err + } + newWorkload := new(appsv1.Deployment) + err = k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: workload.GetName()}, newWorkload) + if err != nil { + return fmt.Errorf("generate same namespace workload error") + } + return nil + }, time.Second*60, time.Microsecond*300).Should(BeNil()) + }) }) func generateResourceTrackerKey(namespace string, name string) types.NamespacedName {