From e320a4b027dac5aaf755071a4e88451622fbbb7d Mon Sep 17 00:00:00 2001 From: yangsoon Date: Tue, 8 Jun 2021 10:30:00 +0800 Subject: [PATCH] add more options for vela-controller (#1769) * add more option for controller 1. add ConcurrentReconciles for setting the concurrent reconcile number of the controller 2. add DependCheckWait for setting the time to wait for ApplicationConfiguration's dependent-resource ready * fix test * add controller reference --- Makefile | 2 +- .../templates/kubevela-controller.yaml | 2 ++ charts/vela-core/values.yaml | 6 +++++ cmd/core/main.go | 6 +++-- .../api/vela-controller-params-reference.md | 24 +++++++++++++++++++ pkg/commands/system.go | 5 ++++ .../core.oam.dev/oamruntime_controller.go | 6 +++++ .../applicationconfiguration.go | 22 +++++++++++++---- .../applicationconfiguration_test.go | 17 ++++++++++++- 9 files changed, 82 insertions(+), 8 deletions(-) create mode 100644 design/api/vela-controller-params-reference.md diff --git a/Makefile b/Makefile index 9e13491cc..1d2078283 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,7 @@ docker-push: docker push ${IMG} e2e-setup: - bin/vela install --set installCertManager=true --image-pull-policy IfNotPresent --image-repo vela-core-test --image-tag $(GIT_COMMIT) + bin/vela install --set installCertManager=true --image-pull-policy IfNotPresent --image-repo vela-core-test --image-tag $(GIT_COMMIT) --depend-check-wait 10s ginkgo version ginkgo -v -r e2e/setup kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s diff --git a/charts/vela-core/templates/kubevela-controller.yaml b/charts/vela-core/templates/kubevela-controller.yaml index cf5378c7d..a8fd115cf 100644 --- a/charts/vela-core/templates/kubevela-controller.yaml +++ b/charts/vela-core/templates/kubevela-controller.yaml @@ -117,6 +117,8 @@ spec: {{ if ne .Values.disableCaps "" }} - "--disable-caps={{ .Values.disableCaps }}" {{ end }} + - "--concurrent-reconciles={{ .Values.concurrentReconciles }}" + - "--depend-check-wait={{ .Values.dependCheckWait }}" image: {{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ quote .Values.image.pullPolicy }} resources: diff --git a/charts/vela-core/values.yaml b/charts/vela-core/values.yaml index 28aca19b3..4c8ae3c6a 100644 --- a/charts/vela-core/values.yaml +++ b/charts/vela-core/values.yaml @@ -86,3 +86,9 @@ certificate: caBundle: replace-me systemDefinitionNamespace: vela-system + +# concurrentReconciles is the concurrent reconcile number of the controller +concurrentReconciles: 4 + +# dependCheckWait is the time to wait for ApplicationConfiguration's dependent-resource ready +dependCheckWait: 30s \ No newline at end of file diff --git a/cmd/core/main.go b/cmd/core/main.go index 5a3bb293d..c9ee5546d 100644 --- a/cmd/core/main.go +++ b/cmd/core/main.go @@ -100,11 +100,13 @@ func main() { "custom-revision-hook-url is a webhook url which will let KubeVela core to call with applicationConfiguration and component info and return a customized component revision") flag.StringVar(&disableCaps, "disable-caps", "", "To be disabled builtin capability list.") flag.StringVar(&storageDriver, "storage-driver", driver.LocalDriverName, "Application file save to the storage driver") - flag.DurationVar(&syncPeriod, "informer-re-sync-interval", 5*time.Minute, - "controller shared informer lister full re-sync period") + flag.DurationVar(&syncPeriod, "informer-re-sync-interval", 2*time.Hour, "controller shared informer lister full re-sync period. The default value is 2 hours") flag.StringVar(&oam.SystemDefinitonNamespace, "system-definition-namespace", "vela-system", "define the namespace of the system-level definition") flag.DurationVar(&controllerArgs.LongWait, "long-wait", 1*time.Minute, "long-wait is controller next reconcile interval time like 30s, 2m etc. The default value is 1m,"+ " you can set it to 0 for no reconcile routine after success") + flag.IntVar(&controllerArgs.ConcurrentReconciles, "concurrent-reconciles", 4, "concurrent-reconciles is the concurrent reconcile number of the controller. The default value is 4") + flag.DurationVar(&controllerArgs.DependCheckWait, "depend-check-wait", 30*time.Second, "depend-check-wait is the time to wait for ApplicationConfiguration's dependent-resource ready."+ + "The default value is 30s, which means if dependent resources were not prepared, the ApplicationConfiguration would be reconciled after 30s.") flag.Parse() // setup logging diff --git a/design/api/vela-controller-params-reference.md b/design/api/vela-controller-params-reference.md new file mode 100644 index 000000000..7338d8628 --- /dev/null +++ b/design/api/vela-controller-params-reference.md @@ -0,0 +1,24 @@ +# KubeVela Controller Parameters Reference + +| parameter | type | default | describe | +| :-------------------------: | :----: | :-------------------------------: | :----------------------------------------------------------: | +| use-webhook | bool | false | Enable Admission Webhook | +| use-trait-injector | bool | false | Enable TraitInjector | +| webhook-cert-dir | string | /k8s-webhook-server/serving-certs | Admission webhook cert/key dir. | +| metrics-addr | string | :8080 | The address the metric endpoint binds to. | +| enable-leader-election | bool | false | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. | +| leader-election-namespace | string | "" | Determines the namespace in which the leader election configmap will be created. | +| log-file-path | string | "" | The file to write logs to. | +| log-retain-date | int | 7 | The number of days of logs history to retain. | +| log-compress | bool | true | Enable compression on the rotated logs. | +| revision-limit | int | 50 | revision-limit is the maximum number of revisions that will be maintained. The default value is 50. | +| health-addr | string | :9440 | The address the health endpoint binds to. | +| apply-once-only | string | false | For the purpose of some production environment that workload or trait should not be affected if no spec change, available options: on, off, force. | +| custom-revision-hook-url | string | "" | custom-revision-hook-url is a webhook url which will let KubeVela core to call with applicationConfiguration and component info and return a customized component revision | +| disable-caps | string | "" | To be disabled builtin capability list. | +| storage-driver | string | Local | Application file save to the storage driver | +| informer-re-sync-interval | time | 2h | controller shared informer lister full re-sync period | +| system-definition-namespace | string | vela-system | define the namespace of the system-level definition | +| long-wait | time | 1m | long-wait is controller next reconcile interval time like 30s, 2m etc. The default value is 1m, you can set it to 0 for no reconcile routine after success | +| concurrent-reconciles | int | 4 | concurrent-reconciles is the concurrent reconcile number of the controller. | +| depend-check-wait | time | 30s | depend-check-wait is the time to wait for ApplicationConfiguration's dependent-resource ready. | diff --git a/pkg/commands/system.go b/pkg/commands/system.go index 4d75b3f69..a3a9a5fae 100644 --- a/pkg/commands/system.go +++ b/pkg/commands/system.go @@ -48,6 +48,7 @@ type chartArgs struct { imageRepo string imageTag string imagePullPolicy string + dependCheckWait string more []string } @@ -131,6 +132,7 @@ func NewInstallCommand(c types.Args, chartContent string, ioStreams cmdutil.IOSt flag.StringVarP(&i.chartArgs.imagePullPolicy, "image-pull-policy", "", "", "vela core image pull policy, this will align to chart value image.pullPolicy") flag.StringVarP(&i.chartArgs.imageRepo, "image-repo", "", "", "vela core image repo, this will align to chart value image.repo") flag.StringVarP(&i.chartArgs.imageTag, "image-tag", "", "", "vela core image repo, this will align to chart value image.tag") + flag.StringVarP(&i.chartArgs.dependCheckWait, "depend-check-wait", "", "", "depend-check-wait, this the time to wait for ApplicationConfiguration's dependent-resource ready") flag.StringVarP(&i.waitReady, "wait", "w", "0s", "wait until vela-core is ready to serve, default will not wait") flag.StringSliceVarP(&i.chartArgs.more, "set", "s", []string{}, "arguments for installing vela-core chart") @@ -228,6 +230,9 @@ func (i *initCmd) resolveValues() (map[string]interface{}, error) { if i.chartArgs.imagePullPolicy != "" { valuesConfig = append(valuesConfig, fmt.Sprintf("image.pullPolicy=%s", i.chartArgs.imagePullPolicy)) } + if i.chartArgs.dependCheckWait != "" { + valuesConfig = append(valuesConfig, fmt.Sprintf("dependCheckWait=%s", i.chartArgs.dependCheckWait)) + } valuesConfig = append(valuesConfig, i.chartArgs.more...) for _, val := range valuesConfig { diff --git a/pkg/controller/core.oam.dev/oamruntime_controller.go b/pkg/controller/core.oam.dev/oamruntime_controller.go index 3b7ea470e..c7e4a6f36 100644 --- a/pkg/controller/core.oam.dev/oamruntime_controller.go +++ b/pkg/controller/core.oam.dev/oamruntime_controller.go @@ -52,4 +52,10 @@ type Args struct { // LongWait is controller next reconcile interval time LongWait time.Duration + + // ConcurrentReconciles is the concurrent reconcile number of the controller + ConcurrentReconciles int + + // DependCheckWait is the time to wait for ApplicationConfiguration's dependent-resource ready + DependCheckWait time.Duration } diff --git a/pkg/controller/core.oam.dev/v1alpha2/applicationconfiguration/applicationconfiguration.go b/pkg/controller/core.oam.dev/v1alpha2/applicationconfiguration/applicationconfiguration.go index 774cd3b9b..204a94dd6 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/applicationconfiguration/applicationconfiguration.go +++ b/pkg/controller/core.oam.dev/v1alpha2/applicationconfiguration/applicationconfiguration.go @@ -31,6 +31,7 @@ import ( "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -51,7 +52,6 @@ import ( const ( reconcileTimeout = 1 * time.Minute - dependCheckWait = 10 * time.Second shortWait = 30 * time.Second ) @@ -92,7 +92,12 @@ func Setup(mgr ctrl.Manager, args core.Args, l logging.Logger) error { } name := "oam/" + strings.ToLower(v1alpha2.ApplicationConfigurationGroupKind) - return ctrl.NewControllerManagedBy(mgr). + builder := ctrl.NewControllerManagedBy(mgr) + builder.WithOptions(controller.Options{ + MaxConcurrentReconciles: args.ConcurrentReconciles, + }) + + return builder. Named(name). For(&v1alpha2.ApplicationConfiguration{}). Watches(&source.Kind{Type: &v1alpha2.Component{}}, &ComponentHandler{ @@ -105,7 +110,8 @@ func Setup(mgr ctrl.Manager, args core.Args, l logging.Logger) error { l.WithValues("controller", name), WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name))), WithApplyOnceOnlyMode(args.ApplyMode), - WithLongWaitTime(args.LongWait))) + WithLongWaitTime(args.LongWait), + WithDependCheckWait(args.DependCheckWait))) } // An OAMApplicationReconciler reconciles OAM ApplicationConfigurations by rendering and @@ -122,6 +128,7 @@ type OAMApplicationReconciler struct { postHooks map[string]ControllerHooks applyOnceOnlyMode core.ApplyOnceOnlyMode longWait time.Duration + dependCheckWait time.Duration } // A ReconcilerOption configures a Reconciler. @@ -186,6 +193,13 @@ func WithLongWaitTime(longWait time.Duration) ReconcilerOption { } } +// WithDependCheckWait set depend check wait +func WithDependCheckWait(dependCheckWait time.Duration) ReconcilerOption { + return func(r *OAMApplicationReconciler) { + r.dependCheckWait = dependCheckWait + } +} + // NewReconciler returns an OAMApplicationReconciler that reconciles ApplicationConfigurations // by rendering and instantiating their Components and Traits. func NewReconciler(m ctrl.Manager, dm discoverymapper.DiscoveryMapper, log logging.Logger, o ...ReconcilerOption) *OAMApplicationReconciler { @@ -349,7 +363,7 @@ func (r *OAMApplicationReconciler) Reconcile(req reconcile.Request) (result reco ac.Status.Dependency = v1alpha2.DependencyStatus{} waitTime := r.longWait if len(depStatus.Unsatisfied) != 0 { - waitTime = dependCheckWait + waitTime = r.dependCheckWait ac.Status.Dependency = *depStatus } diff --git a/pkg/controller/core.oam.dev/v1alpha2/applicationconfiguration/applicationconfiguration_test.go b/pkg/controller/core.oam.dev/v1alpha2/applicationconfiguration/applicationconfiguration_test.go index 46030a4a8..ece30dd56 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/applicationconfiguration/applicationconfiguration_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/applicationconfiguration/applicationconfiguration_test.go @@ -183,6 +183,7 @@ func TestReconciler(t *testing.T) { WithRenderer(ComponentRenderFn(func(_ context.Context, _ *v1alpha2.ApplicationConfiguration) ([]Workload, *v1alpha2.DependencyStatus, error) { return nil, &v1alpha2.DependencyStatus{}, errBoom })), + WithDependCheckWait(10 * time.Second), }, }, want: want{ @@ -212,6 +213,7 @@ func TestReconciler(t *testing.T) { WithApplicator(WorkloadApplyFns{ApplyFn: func(_ context.Context, _ []v1alpha2.WorkloadStatus, _ []Workload, _ ...apply.ApplyOption) error { return errBoom }}), + WithDependCheckWait(10 * time.Second), }, }, want: want{ @@ -245,6 +247,7 @@ func TestReconciler(t *testing.T) { WithGarbageCollector(GarbageCollectorFn(func(_ string, _ []v1alpha2.WorkloadStatus, _ []Workload) []unstructured.Unstructured { return []unstructured.Unstructured{*workload} })), + WithDependCheckWait(10 * time.Second), }, }, want: want{ @@ -308,10 +311,11 @@ func TestReconciler(t *testing.T) { WithGarbageCollector(GarbageCollectorFn(func(_ string, _ []v1alpha2.WorkloadStatus, _ []Workload) []unstructured.Unstructured { return []unstructured.Unstructured{*trait} })), + WithDependCheckWait(10 * time.Second), }, }, want: want{ - result: reconcile.Result{RequeueAfter: dependCheckWait}, + result: reconcile.Result{RequeueAfter: 10 * time.Second}, }, }, "FailedPreHook": { @@ -352,6 +356,7 @@ func TestReconciler(t *testing.T) { WithPosthook("postHook", ControllerHooksFn(func(ctx context.Context, ac *v1alpha2.ApplicationConfiguration, logger logging.Logger) (reconcile.Result, error) { return reconcile.Result{RequeueAfter: shortWait}, nil })), + WithDependCheckWait(10 * time.Second), }, }, want: want{ @@ -422,6 +427,7 @@ func TestReconciler(t *testing.T) { WithPosthook("preHookFailed", ControllerHooksFn(func(ctx context.Context, ac *v1alpha2.ApplicationConfiguration, logger logging.Logger) (reconcile.Result, error) { return reconcile.Result{RequeueAfter: 15 * time.Second}, errBoom })), + WithDependCheckWait(10 * time.Second), }, }, want: want{ @@ -472,6 +478,7 @@ func TestReconciler(t *testing.T) { WithPosthook("preHookFailed", ControllerHooksFn(func(ctx context.Context, ac *v1alpha2.ApplicationConfiguration, logger logging.Logger) (reconcile.Result, error) { return reconcile.Result{RequeueAfter: 15 * time.Second}, errBoom })), + WithDependCheckWait(10 * time.Second), }, }, want: want{ @@ -540,6 +547,7 @@ func TestReconciler(t *testing.T) { return reconcile.Result{RequeueAfter: shortWait}, nil })), WithLongWaitTime(1 * time.Minute), + WithDependCheckWait(10 * time.Second), }, }, want: want{ @@ -584,6 +592,9 @@ func TestReconciler(t *testing.T) { MockStatusUpdate: test.NewMockStatusUpdateFn(nil), }, }, + o: []ReconcilerOption{ + WithDependCheckWait(10 * time.Second), + }, }, want: want{ result: reconcile.Result{}, @@ -616,6 +627,9 @@ func TestReconciler(t *testing.T) { MockStatusUpdate: test.NewMockStatusUpdateFn(nil), }, }, + o: []ReconcilerOption{ + WithDependCheckWait(10 * time.Second), + }, }, want: want{ result: reconcile.Result{}, @@ -652,6 +666,7 @@ func TestReconciler(t *testing.T) { WithApplicator(WorkloadApplyFns{FinalizeFn: func(ctx context.Context, ac *v1alpha2.ApplicationConfiguration) error { return errBoom }}), + WithDependCheckWait(10 * time.Second), }, }, want: want{