diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/revision.go b/pkg/controller/core.oam.dev/v1alpha2/application/revision.go index 28de6cb32..bf34d8793 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/revision.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/revision.go @@ -33,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + ktypes "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" @@ -816,6 +817,64 @@ func (h historiesByRevision) Less(i, j int) bool { } func cleanUpComponentRevision(ctx context.Context, h *AppHandler) error { + if appWillRollout(h.app) { + return cleanUpRollOutComponentRevision(ctx, h) + } + return cleanUpWorkflowComponentRevision(ctx, h) +} + +func cleanUpWorkflowComponentRevision(ctx context.Context, h *AppHandler) error { + // collect component revision in use + compRevisionInUse := map[string]map[string]struct{}{} + for _, resource := range h.app.Status.AppliedResources { + compName := resource.Name + ns := resource.Namespace + r := &unstructured.Unstructured{} + r.GetObjectKind().SetGroupVersionKind(resource.GroupVersionKind()) + err := h.r.Get(ctx, ktypes.NamespacedName{Name: compName, Namespace: ns}, r) + if err != nil { + return err + } + if compRevisionInUse[compName] == nil { + compRevisionInUse[compName] = map[string]struct{}{} + } + compRevision, ok := r.GetLabels()[oam.LabelAppComponentRevision] + if ok { + compRevisionInUse[compName][compRevision] = struct{}{} + } + } + + for _, curComp := range h.app.Status.AppliedResources { + crList := &appsv1.ControllerRevisionList{} + listOpts := []client.ListOption{client.MatchingLabels{ + oam.LabelControllerRevisionComponent: curComp.Name, + }, client.InNamespace(h.app.Namespace)} + if err := h.r.List(ctx, crList, listOpts...); err != nil { + return err + } + needKill := len(crList.Items) - h.r.appRevisionLimit - len(compRevisionInUse[curComp.Name]) + if needKill < 1 { + continue + } + sortedRevision := crList.Items + sort.Sort(historiesByComponentRevision(sortedRevision)) + for _, rev := range sortedRevision { + if needKill <= 0 { + break + } + if _, inUse := compRevisionInUse[curComp.Name][rev.Name]; inUse { + continue + } + if err := h.r.Delete(ctx, rev.DeepCopy()); err != nil && !apierrors.IsNotFound(err) { + return err + } + needKill-- + } + } + return nil +} + +func cleanUpRollOutComponentRevision(ctx context.Context, h *AppHandler) error { appRevInUse, err := gatherUsingAppRevision(ctx, h) if err != nil { return err diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/revision_clean_up_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/revision_clean_up_test.go index 7a2fc851c..2c80a9006 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/revision_clean_up_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/revision_clean_up_test.go @@ -262,6 +262,208 @@ var _ = Describe("Test application controller clean up ", func() { }, time.Second*10, time.Millisecond*500).Should(BeNil()) }) + It("Test clean up rollout component revision", func() { + appName := "app-2" + appKey := types.NamespacedName{Namespace: namespace, Name: appName} + app := getApp(appName, namespace, "normal-worker") + metav1.SetMetaDataAnnotation(&app.ObjectMeta, oam.AnnotationAppRollout, "true") + metav1.SetMetaDataAnnotation(&app.ObjectMeta, oam.AnnotationRollingComponent, "comp1") + Expect(k8sClient.Create(ctx, app)).Should(BeNil()) + checkApp := new(v1beta1.Application) + for i := 0; i < appRevisionLimit+1; i++ { + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + property := fmt.Sprintf(`{"cmd":["sleep","1000"],"image":"busybox:%d"}`, i) + checkApp.Spec.Components[0].Properties = runtime.RawExtension{Raw: []byte(property)} + Expect(k8sClient.Update(ctx, checkApp)).Should(BeNil()) + testutil.ReconcileOnceAfterFinalizer(reconciler, ctrl.Request{NamespacedName: appKey}) + } + listOpts := []client.ListOption{ + client.InNamespace(namespace), + client.MatchingLabels{ + oam.LabelControllerRevisionComponent: "comp1", + }, + } + crList := new(appsv1.ControllerRevisionList) + Eventually(func() error { + err := k8sClient.List(ctx, crList, listOpts...) + if err != nil { + return err + } + if len(crList.Items) != appRevisionLimit+1 { + return fmt.Errorf("error comp revision number wants %d, actually %d", appRevisionLimit+1, len(crList.Items)) + } + return nil + }, time.Second*10, time.Millisecond*500).Should(BeNil()) + + By("create new appRevision will remove revision v1") + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + property := fmt.Sprintf(`{"cmd":["sleep","1000"],"image":"busybox:%d"}`, 6) + checkApp.Spec.Components[0].Properties = runtime.RawExtension{Raw: []byte(property)} + Expect(k8sClient.Update(ctx, checkApp)).Should(BeNil()) + _, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + deletedRevison := new(v1beta1.ApplicationRevision) + revKey := types.NamespacedName{Namespace: namespace, Name: "comp1-v1"} + Eventually(func() error { + if _, err = reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: appKey}); err != nil { + return err + } + err := k8sClient.List(ctx, crList, listOpts...) + if err != nil { + return err + } + if len(crList.Items) != appRevisionLimit+1 { + return fmt.Errorf("error comp revision number wants %d, actually %d", appRevisionLimit+1, len(crList.Items)) + } + err = k8sClient.Get(ctx, revKey, deletedRevison) + if err == nil || !apierrors.IsNotFound(err) { + return fmt.Errorf("haven't clean up the oldest revision") + } + return nil + }, time.Second*10, time.Millisecond*500).Should(BeNil()) + + By("update app again will gc revision v2") + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + property = fmt.Sprintf(`{"cmd":["sleep","1000"],"image":"busybox:%d"}`, 7) + checkApp.Spec.Components[0].Properties = runtime.RawExtension{Raw: []byte(property)} + Expect(k8sClient.Update(ctx, checkApp)).Should(BeNil()) + _, err = reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + revKey = types.NamespacedName{Namespace: namespace, Name: "comp1-v2"} + Eventually(func() error { + if _, err = reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: appKey}); err != nil { + return err + } + err := k8sClient.List(ctx, crList, listOpts...) + if err != nil { + return err + } + if len(crList.Items) != appRevisionLimit+1 { + return fmt.Errorf("error comp revision number wants %d, actually %d", appRevisionLimit+1, len(crList.Items)) + } + err = k8sClient.Get(ctx, revKey, deletedRevison) + if err == nil || !apierrors.IsNotFound(err) { + return fmt.Errorf("haven't clean up the oldest revision") + } + return nil + }, time.Second*10, time.Millisecond*500).Should(BeNil()) + + By("update app with comp as latest revision will not gc revision v3") + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + property = fmt.Sprintf(`{"cmd":["sleep","1000"],"image":"busybox:%d"}`, 6) + checkApp.Spec.Components[0].Properties = runtime.RawExtension{Raw: []byte(property)} + Expect(k8sClient.Update(ctx, checkApp)).Should(BeNil()) + _, err = reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + revKey = types.NamespacedName{Namespace: namespace, Name: "comp1-v3"} + Eventually(func() error { + if _, err = reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: appKey}); err != nil { + return err + } + err := k8sClient.List(ctx, crList, listOpts...) + if err != nil { + return err + } + if len(crList.Items) != appRevisionLimit+1 { + return fmt.Errorf("error comp revision number wants %d, actually %d", appRevisionLimit+1, len(crList.Items)) + } + return k8sClient.Get(ctx, revKey, &appsv1.ControllerRevision{}) + }, time.Second*10, time.Millisecond*500).Should(BeNil()) + }) + + It("Test clean up rollout appRevision", func() { + appName := "app-2" + appKey := types.NamespacedName{Namespace: namespace, Name: appName} + app := getApp(appName, namespace, "normal-worker") + metav1.SetMetaDataAnnotation(&app.ObjectMeta, oam.AnnotationAppRollout, "true") + metav1.SetMetaDataAnnotation(&app.ObjectMeta, oam.AnnotationRollingComponent, "comp1") + Expect(k8sClient.Create(ctx, app)).Should(BeNil()) + checkApp := new(v1beta1.Application) + for i := 0; i < appRevisionLimit+1; i++ { + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + property := fmt.Sprintf(`{"cmd":["sleep","1000"],"image":"busybox:%d"}`, i) + checkApp.Spec.Components[0].Properties = runtime.RawExtension{Raw: []byte(property)} + Expect(k8sClient.Update(ctx, checkApp)).Should(BeNil()) + testutil.ReconcileOnceAfterFinalizer(reconciler, ctrl.Request{NamespacedName: appKey}) + } + listOpts := []client.ListOption{ + client.InNamespace(namespace), + client.MatchingLabels{ + oam.LabelAppName: appName, + }, + } + appRevisionList := new(v1beta1.ApplicationRevisionList) + Eventually(func() error { + err := k8sClient.List(ctx, appRevisionList, listOpts...) + if err != nil { + return err + } + if len(appRevisionList.Items) != appRevisionLimit+1 { + return fmt.Errorf("error appRevison number wants %d, actually %d", appRevisionLimit+1, len(appRevisionList.Items)) + } + return nil + }, time.Second*30, time.Microsecond*300).Should(BeNil()) + + By("create new appRevision will remove appRevison1") + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + property := fmt.Sprintf(`{"cmd":["sleep","1000"],"image":"busybox:%d"}`, 6) + checkApp.Spec.Components[0].Properties = runtime.RawExtension{Raw: []byte(property)} + Expect(k8sClient.Update(ctx, checkApp)).Should(BeNil()) + _, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + deletedRevison := new(v1beta1.ApplicationRevision) + revKey := types.NamespacedName{Namespace: namespace, Name: appName + "-v1"} + Eventually(func() error { + if _, err = reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: appKey}); err != nil { + return err + } + err := k8sClient.List(ctx, appRevisionList, listOpts...) + if err != nil { + return err + } + if len(appRevisionList.Items) != appRevisionLimit+1 { + return fmt.Errorf("error appRevison number wants %d, actually %d", appRevisionLimit+1, len(appRevisionList.Items)) + } + err = k8sClient.Get(ctx, revKey, deletedRevison) + if err == nil || !apierrors.IsNotFound(err) { + return fmt.Errorf("haven't clean up the oldest revision") + } + if res, err := util.CheckAppRevision(appRevisionList.Items, []int{2, 3, 4, 5, 6, 7}); err != nil || !res { + return fmt.Errorf("appRevision collection mismatch") + } + return nil + }, time.Second*10, time.Second*2).Should(BeNil()) + + By("update app again will gc appRevision2") + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + property = fmt.Sprintf(`{"cmd":["sleep","1000"],"image":"busybox:%d"}`, 7) + checkApp.Spec.Components[0].Properties = runtime.RawExtension{Raw: []byte(property)} + Expect(k8sClient.Update(ctx, checkApp)).Should(BeNil()) + _, err = reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: appKey}) + Expect(err).Should(BeNil()) + Eventually(func() error { + if _, err = reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: appKey}); err != nil { + return err + } + err := k8sClient.List(ctx, appRevisionList, listOpts...) + if err != nil { + return err + } + if len(appRevisionList.Items) != appRevisionLimit+1 { + return fmt.Errorf("error appRevison number wants %d, actually %d", appRevisionLimit+1, len(appRevisionList.Items)) + } + revKey = types.NamespacedName{Namespace: namespace, Name: appName + "-v2"} + err = k8sClient.Get(ctx, revKey, deletedRevison) + if err == nil || !apierrors.IsNotFound(err) { + return fmt.Errorf("haven't clean up the revision-2") + } + if res, err := util.CheckAppRevision(appRevisionList.Items, []int{3, 4, 5, 6, 7, 8}); err != nil || !res { + return fmt.Errorf("appRevision collection mismatch") + } + return nil + }, time.Second*30, time.Microsecond*300).Should(BeNil()) + }) + It("Test clean up rollout appRevision", func() { appName := "app-2" appKey := types.NamespacedName{Namespace: namespace, Name: appName}