github/kubevela
1
0
Fork 0
mirror of https://github.com/kubevela/kubevela.git synced 2026-08-27 16:17:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #861 from rain18/health

Browse Source 
feature: add healthcheck policy for workload and trait by cue template
This commit is contained in:
Jianbo Sun
2021-01-19 13:23:58 +08:00
committed by GitHub
parent 7524aeee31 f55dd467b9
commit 47f35e6d62
8 changed files with 399 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go
+8
View File
@@ -108,6 +108,14 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
app.Status.SetConditions(readyCondition("Applied"))
applog.Info("check application health status")
// check application health status
if err := handler.healthCheck(appfile); err != nil {
app.Status.SetConditions(errorCondition("HealthCheck", err))
return handler.Err(err)
}
app.Status.SetConditions(readyCondition("HealthCheck"))
app.Status.Phase = v1alpha2.ApplicationRunning
// Gather status of components
var refComps []v1alpha1.TypedReference
pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go
+172 -3
View File
@@ -25,9 +25,8 @@ import (
"net/http/httptest"
"time"
"github.com/google/go-cmp/cmp"
"github.com/crossplane/crossplane-runtime/apis/core/v1alpha1"
"github.com/google/go-cmp/cmp"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
v1 "k8s.io/api/apps/v1"
@@ -41,6 +40,7 @@ import (
"sigs.k8s.io/yaml"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2"
"github.com/oam-dev/kubevela/pkg/oam"
"github.com/oam-dev/kubevela/pkg/oam/util"
)
@@ -576,6 +576,86 @@ var _ = Describe("Test Application Controller", func() {
Expect(k8sClient.Delete(ctx, app)).Should(BeNil())
})
It("app with health policy for workload", func() {
By("change workload and trait definition with health policy")
nwd, owd := &v1alpha2.WorkloadDefinition{}, &v1alpha2.WorkloadDefinition{}
wDDefJson, _ := yaml.YAMLToJSON([]byte(wDDefWithHealthYaml))
Expect(json.Unmarshal(wDDefJson, nwd)).Should(BeNil())
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: "worker"}, owd)).Should(BeNil())
nwd.ResourceVersion = owd.ResourceVersion
Expect(k8sClient.Update(ctx, nwd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
ntd, otd := &v1alpha2.TraitDefinition{}, &v1alpha2.TraitDefinition{}
tDDefJson, _ := yaml.YAMLToJSON([]byte(tDDefWithHealthYaml))
Expect(json.Unmarshal(tDDefJson, ntd)).Should(BeNil())
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: "scaler"}, otd)).Should(BeNil())
ntd.ResourceVersion = otd.ResourceVersion
Expect(k8sClient.Update(ctx, ntd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
expDeployment := getExpDeployment("myweb6")
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "vela-test-with-health",
},
}
appWithTrait.SetNamespace(ns.Name)
Expect(k8sClient.Create(ctx, ns)).Should(BeNil())
app := appWithTrait.DeepCopy()
expDeployment.Name = app.Name
expDeployment.Namespace = ns.Name
expDeployment.Labels[oam.LabelAppName] = app.Name
Expect(k8sClient.Create(ctx, expDeployment)).Should(BeNil())
expectScalerTrait.SetName(app.Name)
expectScalerTrait.SetNamespace(app.Namespace)
expectScalerTrait.SetLabels(map[string]string{
oam.LabelAppName: app.Name,
"trait.oam.dev/type": "scaler",
})
(expectScalerTrait.Object["spec"].(map[string]interface{}))["workloadRef"] = map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"name": app.Name,
}
Expect(k8sClient.Create(ctx, &expectScalerTrait)).Should(BeNil())
By("enrich the status of deployment and scaler trait")
expDeployment.Status.Replicas = 1
expDeployment.Status.ReadyReplicas = 1
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
got := &v1.Deployment{}
Expect(k8sClient.Get(ctx, client.ObjectKey{
Namespace: app.Namespace,
Name: app.Name,
}, got)).Should(BeNil())
expectScalerTrait.Object["status"] = v1alpha1.ConditionedStatus{
Conditions: []v1alpha1.Condition{{
Status: corev1.ConditionTrue,
LastTransitionTime: metav1.Now(),
}},
}
Expect(k8sClient.Status().Update(ctx, &expectScalerTrait)).Should(BeNil())
tGot := &unstructured.Unstructured{}
tGot.SetAPIVersion("core.oam.dev/v1alpha2")
tGot.SetKind("ManualScalerTrait")
Expect(k8sClient.Get(ctx, client.ObjectKey{
Namespace: app.Namespace,
Name: app.Name,
}, tGot)).Should(BeNil())
By("apply appfile")
Expect(k8sClient.Create(ctx, app)).Should(BeNil())
appKey := client.ObjectKey{
Name: app.Name,
Namespace: app.Namespace,
}
reconcileRetry(reconciler, reconcile.Request{NamespacedName: appKey})
By("Check App running successfully")
checkApp := &v1alpha2.Application{}
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
Expect(checkApp.Status.Phase).Should(Equal(v1alpha2.ApplicationRunning))
Expect(k8sClient.Delete(ctx, app)).Should(BeNil())
})
})
func reconcileRetry(r reconcile.Reconciler, req reconcile.Request) {
@@ -655,8 +735,67 @@ spec:
cmd?: [...string]
}
`
wDDefWithHealthYaml = `
apiVersion: core.oam.dev/v1alpha2
kind: WorkloadDefinition
metadata:
name: worker
annotations:
definition.oam.dev/description: "Long-running scalable backend worker without network endpoint"
spec:
definitionRef:
name: deployments.apps
extension:
healthPolicy: |
isHealth: output.status.readyReplicas == output.status.replicas
template: |
output: {
apiVersion: "apps/v1"
kind: "Deployment"
metadata: {
annotations: {
if context["config"] != _|_ {
for _, v in context.config {
"\(v.name)" : v.value
}
}
}
}
spec: {
selector: matchLabels: {
"app.oam.dev/component": context.name
}
template: {
metadata: labels: {
"app.oam.dev/component": context.name
}
spec: {
containers: [{
name: context.name
image: parameter.image
if parameter["cmd"] != _|_ {
command: parameter.cmd
}
}]
}
}
selector:
matchLabels:
"app.oam.dev/component": context.name
}
}
parameter: {
// +usage=Which image would you like to use for your service
// +short=i
image: string
cmd?: [...string]
}
`
tDDefYaml = `
apiVersion: core.oam.dev/v1alpha2
@@ -730,6 +869,36 @@ spec:
}
}
}
`
tDDefWithHealthYaml = `
apiVersion: core.oam.dev/v1alpha2
kind: TraitDefinition
metadata:
annotations:
definition.oam.dev/description: "Manually scale the app"
name: scaler
spec:
appliesToWorkloads:
- webservice
- worker
definitionRef:
name: manualscalertraits.core.oam.dev
workloadRefPath: spec.workloadRef
extension:
healthPolicy: |
isHealth: output.status.conditions[0].status == "True"
template: |-
output: {
apiVersion: "core.oam.dev/v1alpha2"
kind: "ManualScalerTrait"
spec: {
replicaCount: parameter.replicas
}
}
parameter: {
//+short=r
replicas: *1 | int
}
`
)
pkg/controller/core.oam.dev/v1alpha2/application/apply.go
+25
View File
@@ -15,6 +15,8 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2"
"github.com/oam-dev/kubevela/pkg/controller/core.oam.dev/v1alpha2/application/parser"
"github.com/oam-dev/kubevela/pkg/dsl/process"
)
func errorCondition(tpy string, err error) runtimev1alpha1.Condition {
@@ -71,6 +73,29 @@ func (ret *reter) apply(ctx context.Context, ac *v1alpha2.ApplicationConfigurati
return ret.Sync(ctx, ac, comps)
}
func (ret *reter) healthCheck(appfile *parser.Appfile) error {
for _, wl := range appfile.Services {
pCtx := process.NewContext(wl.Name)
if err := wl.EvalContext(pCtx); err != nil {
return err
}
for _, tr := range wl.Traits {
if err := tr.EvalContext(pCtx); err != nil {
return err
}
}
if err := wl.EvalHealth(pCtx, ret.c, appfile.Name); err != nil {
return err
}
for _, trait := range wl.Traits {
if err := trait.EvalHealth(pCtx, ret.c, appfile.Name); err != nil {
return err
}
}
}
return nil
}
// CreateOrUpdateComponent will create if not exist and update if exists.
func CreateOrUpdateComponent(ctx context.Context, client client.Client, comp *v1alpha2.Component) error {
var getc v1alpha2.Component
pkg/controller/core.oam.dev/v1alpha2/application/parser/service.go
+19 -4
View File
@@ -5,6 +5,7 @@ import (
"github.com/pkg/errors"
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2"
"github.com/oam-dev/kubevela/apis/types"
@@ -32,6 +33,7 @@ type Workload struct {
Type string
Params map[string]interface{}
Template string
Health string
Traits []*Trait
Scopes []Scope
}
@@ -54,7 +56,12 @@ func (wl *Workload) GetUserConfigName() string {
// EvalContext eval workload template and set result to context
func (wl *Workload) EvalContext(ctx process.Context) error {
return definition.NewWDTemplater(wl.Name, wl.Template).Params(wl.Params).Complete(ctx)
return definition.NewWDTemplater(wl.Name, wl.Template, "").Params(wl.Params).Complete(ctx)
}
// EvalHealth eval workload health check
func (wl *Workload) EvalHealth(ctx process.Context, client client.Client, name string) error {
return definition.NewWDTemplater(wl.Name, "", wl.Health).Output(ctx, client, name).HealthCheck()
}
// Scope defines the scope of workload
@@ -68,11 +75,17 @@ type Trait struct {
Name string
Params map[string]interface{}
Template string
Health string
}
// EvalContext eval trait template and set result to context
func (trait *Trait) EvalContext(ctx process.Context) error {
return definition.NewTDTemplater(trait.Name, trait.Template).Params(trait.Params).Complete(ctx)
return definition.NewTDTemplater(trait.Name, trait.Template, "").Params(trait.Params).Complete(ctx)
}
// EvalHealth eval trait health check
func (trait *Trait) EvalHealth(ctx process.Context, client client.Client, name string) error {
return definition.NewTDTemplater(trait.Name, "", trait.Health).Output(ctx, client, name).HealthCheck()
}
// Appfile describes application
@@ -120,11 +133,12 @@ func (pser *Parser) parseWorkload(comp v1alpha2.ApplicationComponent) (*Workload
workload.Traits = []*Trait{}
workload.Name = comp.Name
workload.Type = comp.WorkloadType
templ, err := pser.templ(workload.Type, types.TypeWorkload)
templ, health, err := pser.templ(workload.Type, types.TypeWorkload)
if err != nil && !kerrors.IsNotFound(err) {
return nil, errors.WithMessagef(err, "fetch type of %s", comp.Name)
}
workload.Template = templ
workload.Health = health
settings, err := util.RawExtension2Map(&comp.Settings)
if err != nil {
return nil, errors.WithMessagef(err, "fail to parse settings for %s", comp.Name)
@@ -156,7 +170,7 @@ func (pser *Parser) parseWorkload(comp v1alpha2.ApplicationComponent) (*Workload
}
func (pser *Parser) parseTrait(name string, properties map[string]interface{}) (*Trait, error) {
templ, err := pser.templ(name, types.TypeTrait)
templ, health, err := pser.templ(name, types.TypeTrait)
if kerrors.IsNotFound(err) {
return nil, errors.Errorf("trait definition of %s not found", name)
}
@@ -168,5 +182,6 @@ func (pser *Parser) parseTrait(name string, properties map[string]interface{}) (
Name: name,
Params: properties,
Template: templ,
Health: health,
}, nil
}
pkg/controller/core.oam.dev/v1alpha2/application/template/template.go
+22 -18
View File
@@ -23,52 +23,56 @@ func GetHandler(cli defclient.DefinitionClient) Handler {
}
// Handler is template handler type
type Handler func(key string, kind types.CapType) (string, error)
type Handler func(key string, kind types.CapType) (string, string, error)
// Kind is template kind
type Kind = types.CapType
// LoadTemplate Get template according to key
func (m *manager) LoadTemplate(key string, kd types.CapType) (string, error) {
func (m *manager) LoadTemplate(key string, kd types.CapType) (string, string, error) {
switch kd {
case types.TypeWorkload:
wd, err := m.GetWorkloadDefinition(key)
if err != nil {
return "", errors.WithMessagef(err, "LoadTemplate [%s] ", key)
return "", "", errors.WithMessagef(err, "LoadTemplate [%s] ", key)
}
jsonRaw, err := getTemplate(wd.Spec.Extension.Raw)
tmpl, health, err := getTemplAndHealth(wd.Spec.Extension.Raw)
if err != nil {
return "", errors.WithMessagef(err, "LoadTemplate [%s] ", key)
return "", "", errors.WithMessagef(err, "LoadTemplate [%s] ", key)
}
if jsonRaw == "" {
return "", errors.New("no template found in definition")
if tmpl == "" {
return "", "", errors.New("no template found in definition")
}
return jsonRaw, nil
return tmpl, health, nil
case types.TypeTrait:
td, err := m.GetTraitDefinition(key)
if err != nil {
return "", errors.WithMessagef(err, "LoadTemplate [%s] ", key)
return "", "", errors.WithMessagef(err, "LoadTemplate [%s] ", key)
}
jsonRaw, err := getTemplate(td.Spec.Extension.Raw)
tmpl, health, err := getTemplAndHealth(td.Spec.Extension.Raw)
if err != nil {
return "", errors.WithMessagef(err, "LoadTemplate [%s] ", key)
return "", "", errors.WithMessagef(err, "LoadTemplate [%s] ", key)
}
if jsonRaw == "" {
return "", errors.New("no template found in definition")
if tmpl == "" {
return "", "", errors.New("no template found in definition")
}
return jsonRaw, nil
return tmpl, health, nil
case types.TypeScope:
// TODO: add scope template support
}
return "", fmt.Errorf("kind(%s) of %s not supported", kd, key)
return "", "", fmt.Errorf("kind(%s) of %s not supported", kd, key)
}
func getTemplate(raw []byte) (string, error) {
func getTemplAndHealth(raw []byte) (string, string, error) {
_tmp := map[string]interface{}{}
if err := json.Unmarshal(raw, &_tmp); err != nil {
return "", err
return "", "", err
}
return fmt.Sprint(_tmp["template"]), nil
var health string
if _, ok := _tmp["healthPolicy"]; ok {
health = fmt.Sprint(_tmp["healthPolicy"])
}
return fmt.Sprint(_tmp["template"]), health, nil
}
pkg/controller/core.oam.dev/v1alpha2/application/template/template_test.go
+1 -1
View File
@@ -79,7 +79,7 @@ spec:
m := manager{
mock,
}
temp, err := m.LoadTemplate("worker", types.TypeWorkload)
temp, _, err := m.LoadTemplate("worker", types.TypeWorkload)
if err != nil {
t.Error(err)
return
pkg/dsl/definition/template.go
+149 -4
View File
@@ -1,6 +1,7 @@
package definition
import (
"context"
"encoding/json"
"fmt"
@@ -9,21 +10,35 @@ import (
"cuelang.org/go/cue"
"cuelang.org/go/cue/build"
"github.com/pkg/errors"
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/oam-dev/kubevela/pkg/dsl/model"
"github.com/oam-dev/kubevela/pkg/dsl/process"
"github.com/oam-dev/kubevela/pkg/oam"
)
var (
metadataAccessor = meta.NewAccessor()
)
// Template defines Definition's Render interface
type Template interface {
Params(params interface{}) Template
Complete(ctx process.Context) error
Output(ctx process.Context, client client.Client, name string) Template
HealthCheck() error
}
type def struct {
name string
templ string
health string
params interface{}
output map[string]interface{}
}
type workloadDef struct {
@@ -31,12 +46,14 @@ type workloadDef struct {
}
// NewWDTemplater create Workload Definition templater
func NewWDTemplater(name, templ string) Template {
func NewWDTemplater(name, templ, health string) Template {
return &workloadDef{
def: def{
name: name,
templ: templ,
health: health,
params: nil,
output: nil,
},
}
}
@@ -78,16 +95,65 @@ func (wd *workloadDef) Complete(ctx process.Context) error {
return nil
}
// Output fetch the workload cr and set result to context
func (wd *workloadDef) Output(ctx process.Context, client client.Client, name string) Template {
base, _ := ctx.Output()
componentWorkload, err := base.Unstructured()
if err != nil {
return wd
}
workloadCr, err := getObj(client, componentWorkload, name)
if err != nil {
return wd
}
wd.output = workloadCr
return wd
}
// HealthCheck address health check for workload
func (wd *workloadDef) HealthCheck() error {
if wd.health == "" {
return nil
}
bi := build.NewContext().NewInstance("", nil)
if err := bi.AddFile("-", wd.health); err != nil {
return err
}
if wd.output != nil {
bt, _ := json.Marshal(wd.output)
if err := bi.AddFile("output", fmt.Sprintf("output: %s", string(bt))); err != nil {
return err
}
} else {
return errors.WithMessagef(errors.New("there is no workload output cr for health check"), "workload %s health check", wd.name)
}
insts := cue.Build([]*build.Instance{bi})
for _, inst := range insts {
if err := inst.Value().Err(); err != nil {
return errors.WithMessagef(err, "workload %s check", wd.name)
}
isHealthVal := inst.Lookup("isHealth")
if isHealthVal.Exists() {
healthRs := isHealthVal.Eval()
if isHealth, err := healthRs.Bool(); err != nil || !isHealth {
return errors.WithMessage(err, "the workload is unhealthy")
}
}
}
return nil
}
type traitDef struct {
def
}
// NewTDTemplater create Trait Definition templater
func NewTDTemplater(name, templ string) Template {
func NewTDTemplater(name, templ, health string) Template {
return &traitDef{
def: def{
name: name,
templ: templ,
name: name,
templ: templ,
health: health,
},
}
}
@@ -170,3 +236,82 @@ func (td *traitDef) Complete(ctx process.Context) error {
}
return nil
}
// Output fetch the trait cr and set result to context
func (td *traitDef) Output(ctx process.Context, client client.Client, name string) Template {
_, assists := ctx.Output()
for _, assist := range assists {
if assist.Type != td.name {
continue
}
traitRef, err := assist.Ins.Unstructured()
if err != nil {
return td
}
traitCr, err := getObj(client, traitRef, name)
if err != nil {
return td
}
td.output = traitCr
return td
}
return td
}
// HealthCheck address health check for trait
func (td *traitDef) HealthCheck() error {
if td.health == "" {
return nil
}
bi := build.NewContext().NewInstance("", nil)
if err := bi.AddFile("-", td.health); err != nil {
return err
}
if td.output != nil {
bt, _ := json.Marshal(td.output)
if err := bi.AddFile("output", fmt.Sprintf("output: %s", string(bt))); err != nil {
return err
}
} else {
return errors.WithMessagef(errors.New("there is no trait output cr for health check"), "trait %s health check", td.name)
}
insts := cue.Build([]*build.Instance{bi})
for _, inst := range insts {
if err := inst.Value().Err(); err != nil {
return errors.WithMessagef(err, "trait %s check", td.name)
}
isHealthVal := inst.Lookup("isHealth")
if isHealthVal.Exists() {
if isHealth, err := isHealthVal.Bool(); err != nil || !isHealth {
return errors.WithMessage(err, "the trait is unhealthy")
}
}
}
return nil
}
func getObj(cli client.Client, obj runtime.Object, name string) (map[string]interface{}, error) {
var kind, apiVersion string
var err error
kind, err = metadataAccessor.Kind(obj)
if err != nil {
return nil, fmt.Errorf("cannot access object kind")
}
apiVersion, err = metadataAccessor.APIVersion(obj)
if err != nil {
return nil, fmt.Errorf("cannot access object kind")
}
unList := &unstructured.UnstructuredList{}
unList.SetKind(kind)
unList.SetAPIVersion(apiVersion)
if err := cli.List(context.Background(), unList, client.MatchingLabels{oam.LabelAppName: name}); err != nil {
if kerrors.IsNotFound(err) {
return nil, nil
}
return nil, err
}
if len(unList.Items) == 0 {
return nil, nil
}
return unList.Items[0].Object, nil
}
pkg/dsl/definition/template_test.go
+3 -3
View File
@@ -39,7 +39,7 @@ parameter: {
for _, v := range testCases {
ctx := process.NewContext("test")
wt := NewWDTemplater("-", v.templ)
wt := NewWDTemplater("-", v.templ, "")
if err := wt.Params(v.params).Complete(ctx); err != nil {
t.Error(err)
return
@@ -74,7 +74,7 @@ parameter: {
}
`
ctx := process.NewContext("test")
wt := NewWDTemplater("-", baseTemplate)
wt := NewWDTemplater("-", baseTemplate, "")
if err := wt.Params(map[string]interface{}{
"replicas": 2,
}).Complete(ctx); err != nil {
@@ -107,7 +107,7 @@ parameter: {
}
for _, v := range tds {
td := NewTDTemplater("-", v.templ)
td := NewTDTemplater("-", v.templ, "")
if err := td.Params(v.params).Complete(ctx); err != nil {
t.Error(err)
return