From a89bb69a62d156d520ff8a6ba1fa85cbc93d40ad Mon Sep 17 00:00:00 2001 From: Somefive Date: Mon, 13 Dec 2021 19:41:42 +0800 Subject: [PATCH] Fix: add design docs for ResourceTracker (#2909) * Fix: enhance rt logic and add docs Signed-off-by: Yin Da * Fix: test conflict Signed-off-by: Yin Da --- design/vela-core/resourcetracker_design.md | 56 +++++++++++++ .../apply-once-policy/apply-once.md | 31 ++++++++ .../gc-policy/keep-legacy-resources.md} | 2 +- .../gc-policy/persist-resources.md | 79 +++++++++++++++++++ e2e/application/application_test.go | 10 +-- .../application/application_controller.go | 2 +- pkg/resourcekeeper/statekeep.go | 2 +- test/e2e-test/resource_policy_test.go | 8 +- .../testdata/app/app_garbage_collect.yaml | 6 +- 9 files changed, 184 insertions(+), 12 deletions(-) create mode 100644 design/vela-core/resourcetracker_design.md create mode 100644 docs/examples/app-with-policy/apply-once-policy/apply-once.md rename docs/examples/{gc-policy/gc.md => app-with-policy/gc-policy/keep-legacy-resources.md} (98%) create mode 100644 docs/examples/app-with-policy/gc-policy/persist-resources.md diff --git a/design/vela-core/resourcetracker_design.md b/design/vela-core/resourcetracker_design.md new file mode 100644 index 000000000..ec6007070 --- /dev/null +++ b/design/vela-core/resourcetracker_design.md @@ -0,0 +1,56 @@ +# ResourceTracker: Managing Resources behind Application + +- Owner: Da Yin (@somefive) +- Reviewer: Jian Li (@leejanee), Jianbo Sun (@wonderflow) +- Date: 12/10/2021 +- Status: Implemented + +## Intro + +As the release of Workflow in KubeVela v1.1, resources dispatched by application can be extremely dynamic. +Instead of simply declaring in Application Component, users can have advanced control for applied resources by leveraging WorkflowSteps such as `ApplyObject` or operators such as `op.Delete`. +Meanwhile, with EnvBinding Policy also released in KubeVela 1.1, users can have multiple same components but deployed in different clusters. +These techniques raise new challenges for tracking and maintaining of resources. + +## Goal + +To tackle these challenges, a new architecture of ResourceTracker is proposed and implemented. +Generally, there are several major technical changes compared to the previous version: + +1. Resources do not use OwnerReference to track their ResourceTracker anymore. In other word, the previous bi-directional binding is simplified into uni-directional, which allows us to have more flexible resource management strategies (such as releasing the control of resources). +2. Resources rendered manifests are recorded in ResourceTracker optionally. Based on that, we can prevent configuration drift by leveraging the reconciling mechanism of the Kubernetes operator pattern. +3. ResourceTracker deletion now use finalizer and leverage ApplicationController to reconcile. This ensures the deletion of Application truly removes all managed resources. Also, it allows users to manage versioned resource manually. +4. ResourceTracker in the HubCluster can track resources in ManagedClusters, so that no ResourceTracker is needed anymore in ManagedCluster. Now we can use caches for ResourceTracker again. Additionally, we do not individual multicluster garbage-collect logic anymore. + +From the perspective of users, the direct new-incoming capabilities include: +1. Users can prevent configuration drift by default, which is a common usage of the classical Application model. Alternatively, they can only dispatch resources by leveraging [ApplyOnce](../../docs/examples/app-with-policy/apply-once-policy) Policy, which is the mode of Application-as-Workflow. +2. Users can have customized life-cycle control for application resources by leveraging [GarbageCollect](../../docs/examples/app-with-policy/gc-policy) Policy. For example, users might want to keep resources after version updates or application removal. + +## Implementations + +### ResourceTracker Types + +There are several *ResourceTrackers* maintained for one Application. +- **Versioned ResourceTracker**: Each ResourceTracker keeps the record for the resources of one Application generation. Most resources are kept here. When application spec is updated, new versioned ResourceTracker will be created and used. +- **Root ResourceTracker**: This ResourceTracker keeps the record of the resources that shares the life-cycle with the Application instead of a single version. Resources recorded here will not be recycled until Application is deleted. +- **ControllerRevision ResourceTracker**: This ResourceTracker tracks all the dispatched component revisions. When some components are not in use in new versions, this ResourceTracker can elegantly recycle the revisions for those components. + +### ResourceKeeper + +The main implementation of the resource management logic locates at [pkg/resourcekeeper](../../pkg/resourcekeeper). +The **ResourceKeeper** takes charge of the dispatching, tracking, and recycling of all resources. +- **Dispatch**: First record resources in **ResourceTracker**, then apply resources. Depending on the life-cycle of resources, either **Versioned ResourceTracker** or **Root ResourceTracker** will be used. +- **Delete**: First mark resources as deleted in **ResourceTracker**, then delete resources. +- **StateKeep**: Ranging over all managed resources in the latest **Versioned ResourceTracker** and **Root ResourceTracker**, re-apply those resources. +- **GarbageCollect**: Mark outdated or unused ResourceTrackers as deleted and garbage-collect their managed resources. Details will be delivered below. + +### Garbage Collection Details + +The **GarbageCollect** process includes several steps. +0. **Init**: Scanning over all managed resources in all **Versioned ResourceTrackers** and **Root ResourceTracker** (do not retrieve content from APIServer), aggregating the trackers of each resource and calculate which one RT is responsible for garbage collecting it. +1. **Mark Stage**: Ranging over all ResourceTrackers. If `KeepLegacyResources` is not enabled, outdated ResourceTrackers will be marked as deleted. If enabled, inactive ResourceTrackers, that have all managed resources removed or managed by newer ResourceTrackers, will be marked as deleted. +2. **Sweep Stage**: For all ResourceTrackers marked as deleted, check if all inactive managed resources (managed by newer RT or deleted) are removed (do not exist). If true, remove the finalizer of the ResourceTracker (truly remove it). +3. **Finalize Stage**: For all ResourceTrackers marked as deleted, deleting all inactive managed resources. +4. **GarbageCollectComponentRevisionResourceTracker**: Ranging over all resources in active ResourceTrackers and calculate the component usage. For ComponentRevisions whose component is not in-use anymore, remove them. + +The **Mark Stage** and **GarbageCollectComponentRevisionResourceTracker** will only run when application workflow succeeded, which means when application is still running workflow or new release is not successful, outdated ResourceTrackers will not be marked and resources will not be recycled. diff --git a/docs/examples/app-with-policy/apply-once-policy/apply-once.md b/docs/examples/app-with-policy/apply-once-policy/apply-once.md new file mode 100644 index 000000000..8d2c5988a --- /dev/null +++ b/docs/examples/app-with-policy/apply-once-policy/apply-once.md @@ -0,0 +1,31 @@ +# How to use ApplyOnce policy + +By default, the KubeVela operator will prevent configuration drift for applied resources by reconciling them routinely. This is useful if you want to keep your application always have the desired configuration in avoid of some unintentional changes by external modifiers. + +However, sometimes, you might want to use KubeVela Application to do the dispatch job and recycle job but want to leave resources mutable after workflow is finished. In this case, you can use the following ApplyOnce policy. + +```shell +$ cat < 8000/TCP 78s +``` + +If you upgrade the application and use a different component, you will find the old versioned deployment is deleted by the service is kept. +```shell +$ cat < 8000/TCP 5m56s +hello-world-new ClusterIP 10.96.20.4 8000/TCP 13s +``` diff --git a/e2e/application/application_test.go b/e2e/application/application_test.go index b162e7f2d..c71d80a6b 100644 --- a/e2e/application/application_test.go +++ b/e2e/application/application_test.go @@ -216,11 +216,11 @@ var ApplicationDeleteWithForceOptions = func(context string, appName string) boo gomega.Expect(output).To(gomega.ContainSubstring("timed out")) app = new(v1beta1.Application) - gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: appName, Namespace: "default"}, app)).NotTo(gomega.HaveOccurred()) - meta.RemoveFinalizer(app, "test") - gomega.Eventually(func() error { - return k8sClient.Update(ctx, app) - }, time.Second*3, time.Millisecond*300).Should(gomega.BeNil()) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: appName, Namespace: "default"}, app)).Should(gomega.Succeed()) + meta.RemoveFinalizer(app, "test") + g.Expect(k8sClient.Update(ctx, app)).Should(gomega.Succeed()) + }, time.Second*5, time.Millisecond*300).Should(gomega.Succeed()) cli = fmt.Sprintf("vela delete %s --force", appName) output, err = e2e.ExecAndTerminate(cli) diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go index 5d6b1f2d1..720a1ecc2 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go @@ -279,7 +279,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if err := handler.resourceKeeper.StateKeep(ctx); err != nil { logCtx.Error(err, "Failed to run prevent-configuration-drift") r.Recorder.Event(app, event.Warning(velatypes.ReasonFailedStateKeep, err)) - return r.endWithNegativeCondition(logCtx, app, condition.ReconcileError(err), phase) + app.Status.SetConditions(condition.ErrorCondition("StateKeep", err)) } if err := garbageCollection(logCtx, handler); err != nil { logCtx.Error(err, "Failed to run garbage collection") diff --git a/pkg/resourcekeeper/statekeep.go b/pkg/resourcekeeper/statekeep.go index 745350d13..7b44c6880 100644 --- a/pkg/resourcekeeper/statekeep.go +++ b/pkg/resourcekeeper/statekeep.go @@ -39,7 +39,7 @@ func (h *resourceKeeper) StateKeep(ctx context.Context) error { return entry.err } if mr.Deleted { - if entry.exists { + if entry.exists && entry.obj != nil && entry.obj.GetDeletionTimestamp() == nil { if err := h.Client.Delete(multicluster.ContextWithClusterName(ctx, mr.Cluster), entry.obj); err != nil { return errors.Wrapf(err, "failed to delete outdated resource %s in resourcetracker %s", mr.ResourceKey(), rt.Name) } diff --git a/test/e2e-test/resource_policy_test.go b/test/e2e-test/resource_policy_test.go index f6e972f5a..77baeaff4 100644 --- a/test/e2e-test/resource_policy_test.go +++ b/test/e2e-test/resource_policy_test.go @@ -119,9 +119,11 @@ var _ = Describe("Application Resource-Related Policy Tests", func() { }, 30*time.Second).Should(Succeed()) By("upgrade to v3 (new component)") - Expect(k8sClient.Get(ctx, appKey, app)).Should(Succeed()) - app.Spec.Components[0].Name = "hello-world-new" - Expect(k8sClient.Update(ctx, app)).Should(Succeed()) + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, appKey, app)).Should(Succeed()) + app.Spec.Components[0].Name = "hello-world-new" + g.Expect(k8sClient.Update(ctx, app)).Should(Succeed()) + }, 10*time.Second).Should(Succeed()) Eventually(func(g Gomega) { g.Expect(k8sClient.Get(ctx, appKey, app)).Should(Succeed()) g.Expect(app.Status.ObservedGeneration).Should(Equal(app.Generation)) diff --git a/test/e2e-test/testdata/app/app_garbage_collect.yaml b/test/e2e-test/testdata/app/app_garbage_collect.yaml index 3a8a9dfd3..cf5f3b76b 100644 --- a/test/e2e-test/testdata/app/app_garbage_collect.yaml +++ b/test/e2e-test/testdata/app/app_garbage_collect.yaml @@ -21,4 +21,8 @@ spec: - selector: traitTypes: - expose - strategy: onAppDelete \ No newline at end of file + strategy: onAppDelete + - name: apply-once + type: apply-once + properties: + enable: true \ No newline at end of file