From f55dd467b91cea327b113291cd08839ed986c391 Mon Sep 17 00:00:00 2001 From: "zhangrun.zr" Date: Mon, 4 Jan 2021 10:19:07 +0800 Subject: [PATCH] feature: add healthcheck policy for workload and trait by cue template Signed-off-by: zhangrun.zr --- .../application/application_controller.go | 8 + .../application_controller_test.go | 175 +++++++++++++++++- .../v1alpha2/application/apply.go | 25 +++ .../v1alpha2/application/parser/service.go | 23 ++- .../v1alpha2/application/template/template.go | 40 ++-- .../application/template/template_test.go | 2 +- pkg/dsl/definition/template.go | 153 ++++++++++++++- pkg/dsl/definition/template_test.go | 6 +- 8 files changed, 399 insertions(+), 33 deletions(-) diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go index 3160d6e65..9c0a5b3f2 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go @@ -108,6 +108,14 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { app.Status.SetConditions(readyCondition("Applied")) + applog.Info("check application health status") + // check application health status + if err := handler.healthCheck(appfile); err != nil { + app.Status.SetConditions(errorCondition("HealthCheck", err)) + return handler.Err(err) + } + + app.Status.SetConditions(readyCondition("HealthCheck")) app.Status.Phase = v1alpha2.ApplicationRunning // Gather status of components var refComps []v1alpha1.TypedReference diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go index 23e4ed22a..ba692adf9 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go @@ -25,9 +25,8 @@ import ( "net/http/httptest" "time" - "github.com/google/go-cmp/cmp" - "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1" + "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" v1 "k8s.io/api/apps/v1" @@ -41,6 +40,7 @@ import ( "sigs.k8s.io/yaml" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" + "github.com/oam-dev/kubevela/pkg/oam" "github.com/oam-dev/kubevela/pkg/oam/util" ) @@ -576,6 +576,86 @@ var _ = Describe("Test Application Controller", func() { Expect(k8sClient.Delete(ctx, app)).Should(BeNil()) }) + It("app with health policy for workload", func() { + By("change workload and trait definition with health policy") + nwd, owd := &v1alpha2.WorkloadDefinition{}, &v1alpha2.WorkloadDefinition{} + wDDefJson, _ := yaml.YAMLToJSON([]byte(wDDefWithHealthYaml)) + Expect(json.Unmarshal(wDDefJson, nwd)).Should(BeNil()) + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: "worker"}, owd)).Should(BeNil()) + nwd.ResourceVersion = owd.ResourceVersion + Expect(k8sClient.Update(ctx, nwd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + ntd, otd := &v1alpha2.TraitDefinition{}, &v1alpha2.TraitDefinition{} + tDDefJson, _ := yaml.YAMLToJSON([]byte(tDDefWithHealthYaml)) + Expect(json.Unmarshal(tDDefJson, ntd)).Should(BeNil()) + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: "scaler"}, otd)).Should(BeNil()) + ntd.ResourceVersion = otd.ResourceVersion + Expect(k8sClient.Update(ctx, ntd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + + expDeployment := getExpDeployment("myweb6") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vela-test-with-health", + }, + } + appWithTrait.SetNamespace(ns.Name) + Expect(k8sClient.Create(ctx, ns)).Should(BeNil()) + app := appWithTrait.DeepCopy() + expDeployment.Name = app.Name + expDeployment.Namespace = ns.Name + expDeployment.Labels[oam.LabelAppName] = app.Name + Expect(k8sClient.Create(ctx, expDeployment)).Should(BeNil()) + expectScalerTrait.SetName(app.Name) + expectScalerTrait.SetNamespace(app.Namespace) + expectScalerTrait.SetLabels(map[string]string{ + oam.LabelAppName: app.Name, + "trait.oam.dev/type": "scaler", + }) + (expectScalerTrait.Object["spec"].(map[string]interface{}))["workloadRef"] = map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": app.Name, + } + Expect(k8sClient.Create(ctx, &expectScalerTrait)).Should(BeNil()) + + By("enrich the status of deployment and scaler trait") + expDeployment.Status.Replicas = 1 + expDeployment.Status.ReadyReplicas = 1 + Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil()) + got := &v1.Deployment{} + Expect(k8sClient.Get(ctx, client.ObjectKey{ + Namespace: app.Namespace, + Name: app.Name, + }, got)).Should(BeNil()) + expectScalerTrait.Object["status"] = v1alpha1.ConditionedStatus{ + Conditions: []v1alpha1.Condition{{ + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.Now(), + }}, + } + Expect(k8sClient.Status().Update(ctx, &expectScalerTrait)).Should(BeNil()) + tGot := &unstructured.Unstructured{} + tGot.SetAPIVersion("core.oam.dev/v1alpha2") + tGot.SetKind("ManualScalerTrait") + Expect(k8sClient.Get(ctx, client.ObjectKey{ + Namespace: app.Namespace, + Name: app.Name, + }, tGot)).Should(BeNil()) + + By("apply appfile") + Expect(k8sClient.Create(ctx, app)).Should(BeNil()) + appKey := client.ObjectKey{ + Name: app.Name, + Namespace: app.Namespace, + } + reconcileRetry(reconciler, reconcile.Request{NamespacedName: appKey}) + + By("Check App running successfully") + checkApp := &v1alpha2.Application{} + Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil()) + Expect(checkApp.Status.Phase).Should(Equal(v1alpha2.ApplicationRunning)) + + Expect(k8sClient.Delete(ctx, app)).Should(BeNil()) + }) }) func reconcileRetry(r reconcile.Reconciler, req reconcile.Request) { @@ -655,8 +735,67 @@ spec: cmd?: [...string] } +` + wDDefWithHealthYaml = ` +apiVersion: core.oam.dev/v1alpha2 +kind: WorkloadDefinition +metadata: + name: worker + annotations: + definition.oam.dev/description: "Long-running scalable backend worker without network endpoint" +spec: + definitionRef: + name: deployments.apps + extension: + healthPolicy: | + isHealth: output.status.readyReplicas == output.status.replicas + template: | + output: { + apiVersion: "apps/v1" + kind: "Deployment" + metadata: { + annotations: { + if context["config"] != _|_ { + for _, v in context.config { + "\(v.name)" : v.value + } + } + } + } + spec: { + selector: matchLabels: { + "app.oam.dev/component": context.name + } + template: { + metadata: labels: { + "app.oam.dev/component": context.name + } - + spec: { + containers: [{ + name: context.name + image: parameter.image + + if parameter["cmd"] != _|_ { + command: parameter.cmd + } + }] + } + } + + selector: + matchLabels: + "app.oam.dev/component": context.name + } + } + + parameter: { + // +usage=Which image would you like to use for your service + // +short=i + image: string + + cmd?: [...string] + } ` tDDefYaml = ` apiVersion: core.oam.dev/v1alpha2 @@ -730,6 +869,36 @@ spec: } } } +` + tDDefWithHealthYaml = ` +apiVersion: core.oam.dev/v1alpha2 +kind: TraitDefinition +metadata: + annotations: + definition.oam.dev/description: "Manually scale the app" + name: scaler +spec: + appliesToWorkloads: + - webservice + - worker + definitionRef: + name: manualscalertraits.core.oam.dev + workloadRefPath: spec.workloadRef + extension: + healthPolicy: | + isHealth: output.status.conditions[0].status == "True" + template: |- + output: { + apiVersion: "core.oam.dev/v1alpha2" + kind: "ManualScalerTrait" + spec: { + replicaCount: parameter.replicas + } + } + parameter: { + //+short=r + replicas: *1 | int + } ` ) diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/apply.go b/pkg/controller/core.oam.dev/v1alpha2/application/apply.go index 0aab2ad9c..e48d6a069 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/apply.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/apply.go @@ -15,6 +15,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" + "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev/v1alpha2/application/parser" + "github.com/oam-dev/kubevela/pkg/dsl/process" ) func errorCondition(tpy string, err error) runtimev1alpha1.Condition { @@ -71,6 +73,29 @@ func (ret *reter) apply(ctx context.Context, ac *v1alpha2.ApplicationConfigurati return ret.Sync(ctx, ac, comps) } +func (ret *reter) healthCheck(appfile *parser.Appfile) error { + for _, wl := range appfile.Services { + pCtx := process.NewContext(wl.Name) + if err := wl.EvalContext(pCtx); err != nil { + return err + } + for _, tr := range wl.Traits { + if err := tr.EvalContext(pCtx); err != nil { + return err + } + } + if err := wl.EvalHealth(pCtx, ret.c, appfile.Name); err != nil { + return err + } + for _, trait := range wl.Traits { + if err := trait.EvalHealth(pCtx, ret.c, appfile.Name); err != nil { + return err + } + } + } + return nil +} + // CreateOrUpdateComponent will create if not exist and update if exists. func CreateOrUpdateComponent(ctx context.Context, client client.Client, comp *v1alpha2.Component) error { var getc v1alpha2.Component diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/parser/service.go b/pkg/controller/core.oam.dev/v1alpha2/application/parser/service.go index fb5dc5dde..d8b07b63f 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/parser/service.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/parser/service.go @@ -5,6 +5,7 @@ import ( "github.com/pkg/errors" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" "github.com/oam-dev/kubevela/apis/types" @@ -32,6 +33,7 @@ type Workload struct { Type string Params map[string]interface{} Template string + Health string Traits []*Trait Scopes []Scope } @@ -54,7 +56,12 @@ func (wl *Workload) GetUserConfigName() string { // EvalContext eval workload template and set result to context func (wl *Workload) EvalContext(ctx process.Context) error { - return definition.NewWDTemplater(wl.Name, wl.Template).Params(wl.Params).Complete(ctx) + return definition.NewWDTemplater(wl.Name, wl.Template, "").Params(wl.Params).Complete(ctx) +} + +// EvalHealth eval workload health check +func (wl *Workload) EvalHealth(ctx process.Context, client client.Client, name string) error { + return definition.NewWDTemplater(wl.Name, "", wl.Health).Output(ctx, client, name).HealthCheck() } // Scope defines the scope of workload @@ -68,11 +75,17 @@ type Trait struct { Name string Params map[string]interface{} Template string + Health string } // EvalContext eval trait template and set result to context func (trait *Trait) EvalContext(ctx process.Context) error { - return definition.NewTDTemplater(trait.Name, trait.Template).Params(trait.Params).Complete(ctx) + return definition.NewTDTemplater(trait.Name, trait.Template, "").Params(trait.Params).Complete(ctx) +} + +// EvalHealth eval trait health check +func (trait *Trait) EvalHealth(ctx process.Context, client client.Client, name string) error { + return definition.NewTDTemplater(trait.Name, "", trait.Health).Output(ctx, client, name).HealthCheck() } // Appfile describes application @@ -120,11 +133,12 @@ func (pser *Parser) parseWorkload(comp v1alpha2.ApplicationComponent) (*Workload workload.Traits = []*Trait{} workload.Name = comp.Name workload.Type = comp.WorkloadType - templ, err := pser.templ(workload.Type, types.TypeWorkload) + templ, health, err := pser.templ(workload.Type, types.TypeWorkload) if err != nil && !kerrors.IsNotFound(err) { return nil, errors.WithMessagef(err, "fetch type of %s", comp.Name) } workload.Template = templ + workload.Health = health settings, err := util.RawExtension2Map(&comp.Settings) if err != nil { return nil, errors.WithMessagef(err, "fail to parse settings for %s", comp.Name) @@ -156,7 +170,7 @@ func (pser *Parser) parseWorkload(comp v1alpha2.ApplicationComponent) (*Workload } func (pser *Parser) parseTrait(name string, properties map[string]interface{}) (*Trait, error) { - templ, err := pser.templ(name, types.TypeTrait) + templ, health, err := pser.templ(name, types.TypeTrait) if kerrors.IsNotFound(err) { return nil, errors.Errorf("trait definition of %s not found", name) } @@ -168,5 +182,6 @@ func (pser *Parser) parseTrait(name string, properties map[string]interface{}) ( Name: name, Params: properties, Template: templ, + Health: health, }, nil } diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/template/template.go b/pkg/controller/core.oam.dev/v1alpha2/application/template/template.go index 07f993b3c..244302f0c 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/template/template.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/template/template.go @@ -23,52 +23,56 @@ func GetHandler(cli defclient.DefinitionClient) Handler { } // Handler is template handler type -type Handler func(key string, kind types.CapType) (string, error) +type Handler func(key string, kind types.CapType) (string, string, error) // Kind is template kind type Kind = types.CapType // LoadTemplate Get template according to key -func (m *manager) LoadTemplate(key string, kd types.CapType) (string, error) { +func (m *manager) LoadTemplate(key string, kd types.CapType) (string, string, error) { switch kd { case types.TypeWorkload: wd, err := m.GetWorkloadDefinition(key) if err != nil { - return "", errors.WithMessagef(err, "LoadTemplate [%s] ", key) + return "", "", errors.WithMessagef(err, "LoadTemplate [%s] ", key) } - jsonRaw, err := getTemplate(wd.Spec.Extension.Raw) + tmpl, health, err := getTemplAndHealth(wd.Spec.Extension.Raw) if err != nil { - return "", errors.WithMessagef(err, "LoadTemplate [%s] ", key) + return "", "", errors.WithMessagef(err, "LoadTemplate [%s] ", key) } - if jsonRaw == "" { - return "", errors.New("no template found in definition") + if tmpl == "" { + return "", "", errors.New("no template found in definition") } - return jsonRaw, nil + return tmpl, health, nil case types.TypeTrait: td, err := m.GetTraitDefinition(key) if err != nil { - return "", errors.WithMessagef(err, "LoadTemplate [%s] ", key) + return "", "", errors.WithMessagef(err, "LoadTemplate [%s] ", key) } - jsonRaw, err := getTemplate(td.Spec.Extension.Raw) + tmpl, health, err := getTemplAndHealth(td.Spec.Extension.Raw) if err != nil { - return "", errors.WithMessagef(err, "LoadTemplate [%s] ", key) + return "", "", errors.WithMessagef(err, "LoadTemplate [%s] ", key) } - if jsonRaw == "" { - return "", errors.New("no template found in definition") + if tmpl == "" { + return "", "", errors.New("no template found in definition") } - return jsonRaw, nil + return tmpl, health, nil case types.TypeScope: // TODO: add scope template support } - return "", fmt.Errorf("kind(%s) of %s not supported", kd, key) + return "", "", fmt.Errorf("kind(%s) of %s not supported", kd, key) } -func getTemplate(raw []byte) (string, error) { +func getTemplAndHealth(raw []byte) (string, string, error) { _tmp := map[string]interface{}{} if err := json.Unmarshal(raw, &_tmp); err != nil { - return "", err + return "", "", err } - return fmt.Sprint(_tmp["template"]), nil + var health string + if _, ok := _tmp["healthPolicy"]; ok { + health = fmt.Sprint(_tmp["healthPolicy"]) + } + return fmt.Sprint(_tmp["template"]), health, nil } diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/template/template_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/template/template_test.go index d8d4c7e68..ce979d455 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/template/template_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/template/template_test.go @@ -79,7 +79,7 @@ spec: m := manager{ mock, } - temp, err := m.LoadTemplate("worker", types.TypeWorkload) + temp, _, err := m.LoadTemplate("worker", types.TypeWorkload) if err != nil { t.Error(err) return diff --git a/pkg/dsl/definition/template.go b/pkg/dsl/definition/template.go index 5558fd5db..54361a666 100644 --- a/pkg/dsl/definition/template.go +++ b/pkg/dsl/definition/template.go @@ -1,6 +1,7 @@ package definition import ( + "context" "encoding/json" "fmt" @@ -9,21 +10,35 @@ import ( "cuelang.org/go/cue" "cuelang.org/go/cue/build" "github.com/pkg/errors" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/oam-dev/kubevela/pkg/dsl/model" "github.com/oam-dev/kubevela/pkg/dsl/process" + "github.com/oam-dev/kubevela/pkg/oam" +) + +var ( + metadataAccessor = meta.NewAccessor() ) // Template defines Definition's Render interface type Template interface { Params(params interface{}) Template Complete(ctx process.Context) error + Output(ctx process.Context, client client.Client, name string) Template + HealthCheck() error } type def struct { name string templ string + health string params interface{} + output map[string]interface{} } type workloadDef struct { @@ -31,12 +46,14 @@ type workloadDef struct { } // NewWDTemplater create Workload Definition templater -func NewWDTemplater(name, templ string) Template { +func NewWDTemplater(name, templ, health string) Template { return &workloadDef{ def: def{ name: name, templ: templ, + health: health, params: nil, + output: nil, }, } } @@ -78,16 +95,65 @@ func (wd *workloadDef) Complete(ctx process.Context) error { return nil } +// Output fetch the workload cr and set result to context +func (wd *workloadDef) Output(ctx process.Context, client client.Client, name string) Template { + base, _ := ctx.Output() + componentWorkload, err := base.Unstructured() + if err != nil { + return wd + } + workloadCr, err := getObj(client, componentWorkload, name) + if err != nil { + return wd + } + wd.output = workloadCr + return wd +} + +// HealthCheck address health check for workload +func (wd *workloadDef) HealthCheck() error { + if wd.health == "" { + return nil + } + bi := build.NewContext().NewInstance("", nil) + if err := bi.AddFile("-", wd.health); err != nil { + return err + } + if wd.output != nil { + bt, _ := json.Marshal(wd.output) + if err := bi.AddFile("output", fmt.Sprintf("output: %s", string(bt))); err != nil { + return err + } + } else { + return errors.WithMessagef(errors.New("there is no workload output cr for health check"), "workload %s health check", wd.name) + } + insts := cue.Build([]*build.Instance{bi}) + for _, inst := range insts { + if err := inst.Value().Err(); err != nil { + return errors.WithMessagef(err, "workload %s check", wd.name) + } + isHealthVal := inst.Lookup("isHealth") + if isHealthVal.Exists() { + healthRs := isHealthVal.Eval() + if isHealth, err := healthRs.Bool(); err != nil || !isHealth { + return errors.WithMessage(err, "the workload is unhealthy") + } + } + } + return nil +} + type traitDef struct { def } // NewTDTemplater create Trait Definition templater -func NewTDTemplater(name, templ string) Template { +func NewTDTemplater(name, templ, health string) Template { return &traitDef{ def: def{ - name: name, - templ: templ, + name: name, + templ: templ, + health: health, }, } } @@ -170,3 +236,82 @@ func (td *traitDef) Complete(ctx process.Context) error { } return nil } + +// Output fetch the trait cr and set result to context +func (td *traitDef) Output(ctx process.Context, client client.Client, name string) Template { + _, assists := ctx.Output() + for _, assist := range assists { + if assist.Type != td.name { + continue + } + traitRef, err := assist.Ins.Unstructured() + if err != nil { + return td + } + traitCr, err := getObj(client, traitRef, name) + if err != nil { + return td + } + td.output = traitCr + return td + } + return td +} + +// HealthCheck address health check for trait +func (td *traitDef) HealthCheck() error { + if td.health == "" { + return nil + } + bi := build.NewContext().NewInstance("", nil) + if err := bi.AddFile("-", td.health); err != nil { + return err + } + if td.output != nil { + bt, _ := json.Marshal(td.output) + if err := bi.AddFile("output", fmt.Sprintf("output: %s", string(bt))); err != nil { + return err + } + } else { + return errors.WithMessagef(errors.New("there is no trait output cr for health check"), "trait %s health check", td.name) + } + insts := cue.Build([]*build.Instance{bi}) + for _, inst := range insts { + if err := inst.Value().Err(); err != nil { + return errors.WithMessagef(err, "trait %s check", td.name) + } + isHealthVal := inst.Lookup("isHealth") + if isHealthVal.Exists() { + if isHealth, err := isHealthVal.Bool(); err != nil || !isHealth { + return errors.WithMessage(err, "the trait is unhealthy") + } + } + } + return nil +} + +func getObj(cli client.Client, obj runtime.Object, name string) (map[string]interface{}, error) { + var kind, apiVersion string + var err error + kind, err = metadataAccessor.Kind(obj) + if err != nil { + return nil, fmt.Errorf("cannot access object kind") + } + apiVersion, err = metadataAccessor.APIVersion(obj) + if err != nil { + return nil, fmt.Errorf("cannot access object kind") + } + unList := &unstructured.UnstructuredList{} + unList.SetKind(kind) + unList.SetAPIVersion(apiVersion) + if err := cli.List(context.Background(), unList, client.MatchingLabels{oam.LabelAppName: name}); err != nil { + if kerrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + if len(unList.Items) == 0 { + return nil, nil + } + return unList.Items[0].Object, nil +} diff --git a/pkg/dsl/definition/template_test.go b/pkg/dsl/definition/template_test.go index 3d3dbd781..ddaef151b 100644 --- a/pkg/dsl/definition/template_test.go +++ b/pkg/dsl/definition/template_test.go @@ -39,7 +39,7 @@ parameter: { for _, v := range testCases { ctx := process.NewContext("test") - wt := NewWDTemplater("-", v.templ) + wt := NewWDTemplater("-", v.templ, "") if err := wt.Params(v.params).Complete(ctx); err != nil { t.Error(err) return @@ -74,7 +74,7 @@ parameter: { } ` ctx := process.NewContext("test") - wt := NewWDTemplater("-", baseTemplate) + wt := NewWDTemplater("-", baseTemplate, "") if err := wt.Params(map[string]interface{}{ "replicas": 2, }).Complete(ctx); err != nil { @@ -107,7 +107,7 @@ parameter: { } for _, v := range tds { - td := NewTDTemplater("-", v.templ) + td := NewTDTemplater("-", v.templ, "") if err := td.Params(v.params).Complete(ctx); err != nil { t.Error(err) return