From 324fc1ac7c930477a1cea94ab53371624b492a1c Mon Sep 17 00:00:00 2001 From: Yue Wang Date: Fri, 20 Aug 2021 15:55:47 +0800 Subject: [PATCH] add CUE-based health check in HealthScope controller (#1956) add e2e test Signed-off-by: Yue Wang --- .../core.oam.dev/v1alpha2/core_scope_types.go | 43 ++- .../v1alpha2/zz_generated.deepcopy.go | 115 ++++++- .../crds/core.oam.dev_healthscopes.yaml | 313 +++++++++++++++++- .../crds/core.oam.dev_healthscopes.yaml | 313 +++++++++++++++++- .../crds/core.oam.dev_healthscopes.yaml | 306 ++++++++++++++++- pkg/appfile/parser.go | 5 +- .../core/scopes/healthscope/healthscope.go | 175 +++++++++- .../healthscope/healthscope_controller.go | 161 +++++++-- test/e2e-test/health_scope_test.go | 155 +++++++-- .../testdata/app/app_healthscope.yaml | 20 ++ .../app/app_healthscope_unhealthy.yaml | 14 + 11 files changed, 1544 insertions(+), 76 deletions(-) create mode 100644 test/e2e-test/testdata/app/app_healthscope.yaml create mode 100644 test/e2e-test/testdata/app/app_healthscope_unhealthy.yaml diff --git a/apis/core.oam.dev/v1alpha2/core_scope_types.go b/apis/core.oam.dev/v1alpha2/core_scope_types.go index d2930c5f2..844d92675 100644 --- a/apis/core.oam.dev/v1alpha2/core_scope_types.go +++ b/apis/core.oam.dev/v1alpha2/core_scope_types.go @@ -47,10 +47,27 @@ type HealthScopeSpec struct { // ProbeInterval is the amount of time in seconds between probing tries. ProbeInterval *int32 `json:"probe-interval,omitempty"` + // AppRefs records references of applications' components + AppRefs []AppReference `json:"appReferences,omitempty"` + // WorkloadReferences to the workloads that are in this scope. + // +deprecated WorkloadReferences []corev1.ObjectReference `json:"workloadRefs"` } +// AppReference records references of an application's components +type AppReference struct { + AppName string `json:"appName,omitempty"` + CompReferences []CompReference `json:"compReferences,omitempty"` +} + +// CompReference records references of a component's resources +type CompReference struct { + CompName string `json:"compName,omitempty"` + Workload corev1.ObjectReference `json:"workload,omitempty"` + Traits []corev1.ObjectReference `json:"traits,omitempty"` +} + // A HealthScopeStatus represents the observed state of a HealthScope. type HealthScopeStatus struct { condition.ConditionedStatus `json:",inline"` @@ -58,10 +75,21 @@ type HealthScopeStatus struct { // ScopeHealthCondition represents health condition summary of the scope ScopeHealthCondition ScopeHealthCondition `json:"scopeHealthCondition"` + // AppHealthConditions represents health condition of applications in the scope + AppHealthConditions []*AppHealthCondition `json:"appHealthConditions,omitempty"` + // WorkloadHealthConditions represents health condition of workloads in the scope + // Use AppHealthConditions to provide app level status + // +deprecated WorkloadHealthConditions []*WorkloadHealthCondition `json:"healthConditions,omitempty"` } +// AppHealthCondition represents health condition of an application +type AppHealthCondition struct { + AppName string `json:"appName"` + Components []*WorkloadHealthCondition `json:"components,omitempty"` +} + // ScopeHealthCondition represents health condition summary of a scope. type ScopeHealthCondition struct { HealthStatus HealthStatus `json:"healthStatus"` @@ -71,7 +99,7 @@ type ScopeHealthCondition struct { UnknownWorkloads int64 `json:"unknownWorkloads,omitempty"` } -// WorkloadHealthCondition represents informative health condition. +// WorkloadHealthCondition represents informative health condition of a workload. type WorkloadHealthCondition struct { // ComponentName represents the component name if target is a workload ComponentName string `json:"componentName,omitempty"` @@ -79,7 +107,18 @@ type WorkloadHealthCondition struct { HealthStatus HealthStatus `json:"healthStatus"` Diagnosis string `json:"diagnosis,omitempty"` // WorkloadStatus represents status of workloads whose HealthStatus is UNKNOWN. - WorkloadStatus string `json:"workloadStatus,omitempty"` + WorkloadStatus string `json:"workloadStatus,omitempty"` + CustomStatusMsg string `json:"customStatusMsg,omitempty"` + Traits []*TraitHealthCondition `json:"traits,omitempty"` +} + +// TraitHealthCondition represents informative health condition of a trait. +type TraitHealthCondition struct { + Type string `json:"type"` + Resource string `json:"resource"` + HealthStatus HealthStatus `json:"healthStatus"` + Diagnosis string `json:"diagnosis,omitempty"` + CustomStatusMsg string `json:"customStatusMsg,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/core.oam.dev/v1alpha2/zz_generated.deepcopy.go b/apis/core.oam.dev/v1alpha2/zz_generated.deepcopy.go index c22ccc15c..a217130d5 100644 --- a/apis/core.oam.dev/v1alpha2/zz_generated.deepcopy.go +++ b/apis/core.oam.dev/v1alpha2/zz_generated.deepcopy.go @@ -28,6 +28,54 @@ import ( "github.com/oam-dev/kubevela/apis/standard.oam.dev/v1alpha1" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AppHealthCondition) DeepCopyInto(out *AppHealthCondition) { + *out = *in + if in.Components != nil { + in, out := &in.Components, &out.Components + *out = make([]*WorkloadHealthCondition, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(WorkloadHealthCondition) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AppHealthCondition. +func (in *AppHealthCondition) DeepCopy() *AppHealthCondition { + if in == nil { + return nil + } + out := new(AppHealthCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AppReference) DeepCopyInto(out *AppReference) { + *out = *in + if in.CompReferences != nil { + in, out := &in.CompReferences, &out.CompReferences + *out = make([]CompReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AppReference. +func (in *AppReference) DeepCopy() *AppReference { + if in == nil { + return nil + } + out := new(AppReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AppRollout) DeepCopyInto(out *AppRollout) { *out = *in @@ -655,6 +703,27 @@ func (in *CPUResources) DeepCopy() *CPUResources { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CompReference) DeepCopyInto(out *CompReference) { + *out = *in + out.Workload = in.Workload + if in.Traits != nil { + in, out := &in.Traits, &out.Traits + *out = make([]v1.ObjectReference, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CompReference. +func (in *CompReference) DeepCopy() *CompReference { + if in == nil { + return nil + } + out := new(CompReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Component) DeepCopyInto(out *Component) { *out = *in @@ -1654,6 +1723,13 @@ func (in *HealthScopeSpec) DeepCopyInto(out *HealthScopeSpec) { *out = new(int32) **out = **in } + if in.AppRefs != nil { + in, out := &in.AppRefs, &out.AppRefs + *out = make([]AppReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.WorkloadReferences != nil { in, out := &in.WorkloadReferences, &out.WorkloadReferences *out = make([]v1.ObjectReference, len(*in)) @@ -1676,6 +1752,17 @@ func (in *HealthScopeStatus) DeepCopyInto(out *HealthScopeStatus) { *out = *in in.ConditionedStatus.DeepCopyInto(&out.ConditionedStatus) out.ScopeHealthCondition = in.ScopeHealthCondition + if in.AppHealthConditions != nil { + in, out := &in.AppHealthConditions, &out.AppHealthConditions + *out = make([]*AppHealthCondition, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(AppHealthCondition) + (*in).DeepCopyInto(*out) + } + } + } if in.WorkloadHealthConditions != nil { in, out := &in.WorkloadHealthConditions, &out.WorkloadHealthConditions *out = make([]*WorkloadHealthCondition, len(*in)) @@ -1683,7 +1770,7 @@ func (in *HealthScopeStatus) DeepCopyInto(out *HealthScopeStatus) { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] *out = new(WorkloadHealthCondition) - **out = **in + (*in).DeepCopyInto(*out) } } } @@ -2090,6 +2177,21 @@ func (in *TraitDefinitionStatus) DeepCopy() *TraitDefinitionStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TraitHealthCondition) DeepCopyInto(out *TraitHealthCondition) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TraitHealthCondition. +func (in *TraitHealthCondition) DeepCopy() *TraitHealthCondition { + if in == nil { + return nil + } + out := new(TraitHealthCondition) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UnstaifiedDependency) DeepCopyInto(out *UnstaifiedDependency) { *out = *in @@ -2269,6 +2371,17 @@ func (in *WorkloadDefinitionStatus) DeepCopy() *WorkloadDefinitionStatus { func (in *WorkloadHealthCondition) DeepCopyInto(out *WorkloadHealthCondition) { *out = *in out.TargetWorkload = in.TargetWorkload + if in.Traits != nil { + in, out := &in.Traits, &out.Traits + *out = make([]*TraitHealthCondition, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(TraitHealthCondition) + **out = **in + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadHealthCondition. diff --git a/charts/oam-runtime/crds/core.oam.dev_healthscopes.yaml b/charts/oam-runtime/crds/core.oam.dev_healthscopes.yaml index f5fd1e27e..b39d94f33 100644 --- a/charts/oam-runtime/crds/core.oam.dev_healthscopes.yaml +++ b/charts/oam-runtime/crds/core.oam.dev_healthscopes.yaml @@ -42,6 +42,163 @@ spec: spec: description: A HealthScopeSpec defines the desired state of a HealthScope. properties: + appReferences: + description: AppRefs records references of applications' components + items: + description: AppReference records references of an application's + components + properties: + appName: + type: string + compReferences: + items: + description: CompReference records references of a component's + resources + properties: + compName: + type: string + traits: + items: + description: 'ObjectReference contains enough information + to let you inspect or modify the referred object. + --- New uses of this type are discouraged because + of difficulty describing its usage when embedded in + APIs. 1. Ignored fields. It includes many fields + which are not generally honored. For instance, ResourceVersion + and FieldPath are both very rarely valid in actual + usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, + "must refer only to types A and B" or "UID not honored" + or "name must be restricted". Those cannot be + well described when embedded. 3. Inconsistent validation. Because + the usages are different, the validation rules are + different by usage, which makes it hard for users + to predict what will happen. 4. The fields are both + imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during + interpretation and require a REST mapping. In most + cases, the dependency is on the group,resource tuple and + the version of the actual struct is irrelevant. 5. + We cannot easily change it. Because this type is + embedded in many locations, updates to this type will + affect numerous schemas. Don''t make new APIs embed + an underspecified API type they do not control. Instead + of using this type, create a locally provided and + used type that is well-focused on your reference. + For example, ServiceReferences for admission registration: + https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + .' + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object + instead of an entire object, this string should + contain a valid JSON/Go field access statement, + such as desiredState.manifest.containers[2]. For + example, if the object reference is to a container + within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to + the name of the container that triggered the event) + or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax + is chosen only to have some well-defined way of + referencing a part of an object. TODO: this design + is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which + this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: array + workload: + description: 'ObjectReference contains enough information + to let you inspect or modify the referred object. --- + New uses of this type are discouraged because of difficulty + describing its usage when embedded in APIs. 1. Ignored + fields. It includes many fields which are not generally + honored. For instance, ResourceVersion and FieldPath + are both very rarely valid in actual usage. 2. Invalid + usage help. It is impossible to add specific help for + individual usage. In most embedded usages, there are + particular restrictions like, "must refer only to + types A and B" or "UID not honored" or "name must be + restricted". Those cannot be well described when + embedded. 3. Inconsistent validation. Because the + usages are different, the validation rules are different + by usage, which makes it hard for users to predict what + will happen. 4. The fields are both imprecise and overly + precise. Kind is not a precise mapping to a URL. This + can produce ambiguity during interpretation and + require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of + the actual struct is irrelevant. 5. We cannot easily + change it. Because this type is embedded in many locations, + updates to this type will affect numerous schemas. Don''t + make new APIs embed an underspecified API type they + do not control. Instead of using this type, create a + locally provided and used type that is well-focused + on your reference. For example, ServiceReferences for + admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + .' + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object + instead of an entire object, this string should + contain a valid JSON/Go field access statement, + such as desiredState.manifest.containers[2]. For + example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container + that triggered the event) or if no container name + is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part + of an object. TODO: this design is not final and + this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this + reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: object + type: array + type: object + type: array probe-interval: description: ProbeInterval is the amount of time in seconds between probing tries. @@ -122,6 +279,133 @@ spec: status: description: A HealthScopeStatus represents the observed state of a HealthScope. properties: + appHealthConditions: + description: AppHealthConditions represents health condition of applications + in the scope + items: + description: AppHealthCondition represents health condition of an + application + properties: + appName: + type: string + components: + items: + description: WorkloadHealthCondition represents informative + health condition of a workload. + properties: + componentName: + description: ComponentName represents the component name + if target is a workload + type: string + customStatusMsg: + type: string + diagnosis: + type: string + healthStatus: + description: HealthStatus represents health status strings. + type: string + targetWorkload: + description: 'ObjectReference contains enough information + to let you inspect or modify the referred object. --- + New uses of this type are discouraged because of difficulty + describing its usage when embedded in APIs. 1. Ignored + fields. It includes many fields which are not generally + honored. For instance, ResourceVersion and FieldPath + are both very rarely valid in actual usage. 2. Invalid + usage help. It is impossible to add specific help for + individual usage. In most embedded usages, there are + particular restrictions like, "must refer only to + types A and B" or "UID not honored" or "name must be + restricted". Those cannot be well described when + embedded. 3. Inconsistent validation. Because the + usages are different, the validation rules are different + by usage, which makes it hard for users to predict what + will happen. 4. The fields are both imprecise and overly + precise. Kind is not a precise mapping to a URL. This + can produce ambiguity during interpretation and + require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of + the actual struct is irrelevant. 5. We cannot easily + change it. Because this type is embedded in many locations, + updates to this type will affect numerous schemas. Don''t + make new APIs embed an underspecified API type they + do not control. Instead of using this type, create a + locally provided and used type that is well-focused + on your reference. For example, ServiceReferences for + admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + .' + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object + instead of an entire object, this string should + contain a valid JSON/Go field access statement, + such as desiredState.manifest.containers[2]. For + example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container + that triggered the event) or if no container name + is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part + of an object. TODO: this design is not final and + this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this + reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + traits: + items: + description: TraitHealthCondition represents informative + health condition of a trait. + properties: + customStatusMsg: + type: string + diagnosis: + type: string + healthStatus: + description: HealthStatus represents health status + strings. + type: string + resource: + type: string + type: + type: string + required: + - healthStatus + - resource + - type + type: object + type: array + workloadStatus: + description: WorkloadStatus represents status of workloads + whose HealthStatus is UNKNOWN. + type: string + required: + - healthStatus + type: object + type: array + required: + - appName + type: object + type: array conditions: description: Conditions of the resource. items: @@ -157,15 +441,18 @@ spec: type: array healthConditions: description: WorkloadHealthConditions represents health condition - of workloads in the scope + of workloads in the scope Use AppHealthConditions to provide app + level status items: description: WorkloadHealthCondition represents informative health - condition. + condition of a workload. properties: componentName: description: ComponentName represents the component name if target is a workload type: string + customStatusMsg: + type: string diagnosis: type: string healthStatus: @@ -233,6 +520,28 @@ spec: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string type: object + traits: + items: + description: TraitHealthCondition represents informative health + condition of a trait. + properties: + customStatusMsg: + type: string + diagnosis: + type: string + healthStatus: + description: HealthStatus represents health status strings. + type: string + resource: + type: string + type: + type: string + required: + - healthStatus + - resource + - type + type: object + type: array workloadStatus: description: WorkloadStatus represents status of workloads whose HealthStatus is UNKNOWN. diff --git a/charts/vela-core/crds/core.oam.dev_healthscopes.yaml b/charts/vela-core/crds/core.oam.dev_healthscopes.yaml index f5fd1e27e..b39d94f33 100644 --- a/charts/vela-core/crds/core.oam.dev_healthscopes.yaml +++ b/charts/vela-core/crds/core.oam.dev_healthscopes.yaml @@ -42,6 +42,163 @@ spec: spec: description: A HealthScopeSpec defines the desired state of a HealthScope. properties: + appReferences: + description: AppRefs records references of applications' components + items: + description: AppReference records references of an application's + components + properties: + appName: + type: string + compReferences: + items: + description: CompReference records references of a component's + resources + properties: + compName: + type: string + traits: + items: + description: 'ObjectReference contains enough information + to let you inspect or modify the referred object. + --- New uses of this type are discouraged because + of difficulty describing its usage when embedded in + APIs. 1. Ignored fields. It includes many fields + which are not generally honored. For instance, ResourceVersion + and FieldPath are both very rarely valid in actual + usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, + "must refer only to types A and B" or "UID not honored" + or "name must be restricted". Those cannot be + well described when embedded. 3. Inconsistent validation. Because + the usages are different, the validation rules are + different by usage, which makes it hard for users + to predict what will happen. 4. The fields are both + imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during + interpretation and require a REST mapping. In most + cases, the dependency is on the group,resource tuple and + the version of the actual struct is irrelevant. 5. + We cannot easily change it. Because this type is + embedded in many locations, updates to this type will + affect numerous schemas. Don''t make new APIs embed + an underspecified API type they do not control. Instead + of using this type, create a locally provided and + used type that is well-focused on your reference. + For example, ServiceReferences for admission registration: + https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + .' + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object + instead of an entire object, this string should + contain a valid JSON/Go field access statement, + such as desiredState.manifest.containers[2]. For + example, if the object reference is to a container + within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to + the name of the container that triggered the event) + or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax + is chosen only to have some well-defined way of + referencing a part of an object. TODO: this design + is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which + this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: array + workload: + description: 'ObjectReference contains enough information + to let you inspect or modify the referred object. --- + New uses of this type are discouraged because of difficulty + describing its usage when embedded in APIs. 1. Ignored + fields. It includes many fields which are not generally + honored. For instance, ResourceVersion and FieldPath + are both very rarely valid in actual usage. 2. Invalid + usage help. It is impossible to add specific help for + individual usage. In most embedded usages, there are + particular restrictions like, "must refer only to + types A and B" or "UID not honored" or "name must be + restricted". Those cannot be well described when + embedded. 3. Inconsistent validation. Because the + usages are different, the validation rules are different + by usage, which makes it hard for users to predict what + will happen. 4. The fields are both imprecise and overly + precise. Kind is not a precise mapping to a URL. This + can produce ambiguity during interpretation and + require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of + the actual struct is irrelevant. 5. We cannot easily + change it. Because this type is embedded in many locations, + updates to this type will affect numerous schemas. Don''t + make new APIs embed an underspecified API type they + do not control. Instead of using this type, create a + locally provided and used type that is well-focused + on your reference. For example, ServiceReferences for + admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + .' + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object + instead of an entire object, this string should + contain a valid JSON/Go field access statement, + such as desiredState.manifest.containers[2]. For + example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container + that triggered the event) or if no container name + is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part + of an object. TODO: this design is not final and + this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this + reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: object + type: array + type: object + type: array probe-interval: description: ProbeInterval is the amount of time in seconds between probing tries. @@ -122,6 +279,133 @@ spec: status: description: A HealthScopeStatus represents the observed state of a HealthScope. properties: + appHealthConditions: + description: AppHealthConditions represents health condition of applications + in the scope + items: + description: AppHealthCondition represents health condition of an + application + properties: + appName: + type: string + components: + items: + description: WorkloadHealthCondition represents informative + health condition of a workload. + properties: + componentName: + description: ComponentName represents the component name + if target is a workload + type: string + customStatusMsg: + type: string + diagnosis: + type: string + healthStatus: + description: HealthStatus represents health status strings. + type: string + targetWorkload: + description: 'ObjectReference contains enough information + to let you inspect or modify the referred object. --- + New uses of this type are discouraged because of difficulty + describing its usage when embedded in APIs. 1. Ignored + fields. It includes many fields which are not generally + honored. For instance, ResourceVersion and FieldPath + are both very rarely valid in actual usage. 2. Invalid + usage help. It is impossible to add specific help for + individual usage. In most embedded usages, there are + particular restrictions like, "must refer only to + types A and B" or "UID not honored" or "name must be + restricted". Those cannot be well described when + embedded. 3. Inconsistent validation. Because the + usages are different, the validation rules are different + by usage, which makes it hard for users to predict what + will happen. 4. The fields are both imprecise and overly + precise. Kind is not a precise mapping to a URL. This + can produce ambiguity during interpretation and + require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of + the actual struct is irrelevant. 5. We cannot easily + change it. Because this type is embedded in many locations, + updates to this type will affect numerous schemas. Don''t + make new APIs embed an underspecified API type they + do not control. Instead of using this type, create a + locally provided and used type that is well-focused + on your reference. For example, ServiceReferences for + admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + .' + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object + instead of an entire object, this string should + contain a valid JSON/Go field access statement, + such as desiredState.manifest.containers[2]. For + example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container + that triggered the event) or if no container name + is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part + of an object. TODO: this design is not final and + this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this + reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + traits: + items: + description: TraitHealthCondition represents informative + health condition of a trait. + properties: + customStatusMsg: + type: string + diagnosis: + type: string + healthStatus: + description: HealthStatus represents health status + strings. + type: string + resource: + type: string + type: + type: string + required: + - healthStatus + - resource + - type + type: object + type: array + workloadStatus: + description: WorkloadStatus represents status of workloads + whose HealthStatus is UNKNOWN. + type: string + required: + - healthStatus + type: object + type: array + required: + - appName + type: object + type: array conditions: description: Conditions of the resource. items: @@ -157,15 +441,18 @@ spec: type: array healthConditions: description: WorkloadHealthConditions represents health condition - of workloads in the scope + of workloads in the scope Use AppHealthConditions to provide app + level status items: description: WorkloadHealthCondition represents informative health - condition. + condition of a workload. properties: componentName: description: ComponentName represents the component name if target is a workload type: string + customStatusMsg: + type: string diagnosis: type: string healthStatus: @@ -233,6 +520,28 @@ spec: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string type: object + traits: + items: + description: TraitHealthCondition represents informative health + condition of a trait. + properties: + customStatusMsg: + type: string + diagnosis: + type: string + healthStatus: + description: HealthStatus represents health status strings. + type: string + resource: + type: string + type: + type: string + required: + - healthStatus + - resource + - type + type: object + type: array workloadStatus: description: WorkloadStatus represents status of workloads whose HealthStatus is UNKNOWN. diff --git a/legacy/charts/vela-core-legacy/crds/core.oam.dev_healthscopes.yaml b/legacy/charts/vela-core-legacy/crds/core.oam.dev_healthscopes.yaml index 4750e6ee6..a5e65e0b2 100644 --- a/legacy/charts/vela-core-legacy/crds/core.oam.dev_healthscopes.yaml +++ b/legacy/charts/vela-core-legacy/crds/core.oam.dev_healthscopes.yaml @@ -42,6 +42,158 @@ spec: spec: description: A HealthScopeSpec defines the desired state of a HealthScope. properties: + appReferences: + description: AppRefs records references of applications' components + items: + description: AppReference records references of an application's components + properties: + appName: + type: string + compReferences: + items: + description: CompReference records references of a component's + resources + properties: + compName: + type: string + traits: + items: + description: 'ObjectReference contains enough information + to let you inspect or modify the referred object. --- + New uses of this type are discouraged because of difficulty + describing its usage when embedded in APIs. 1. Ignored + fields. It includes many fields which are not generally + honored. For instance, ResourceVersion and FieldPath + are both very rarely valid in actual usage. 2. Invalid + usage help. It is impossible to add specific help for + individual usage. In most embedded usages, there are + particular restrictions like, "must refer only to + types A and B" or "UID not honored" or "name must be + restricted". Those cannot be well described when + embedded. 3. Inconsistent validation. Because the + usages are different, the validation rules are different + by usage, which makes it hard for users to predict what + will happen. 4. The fields are both imprecise and overly + precise. Kind is not a precise mapping to a URL. This + can produce ambiguity during interpretation and + require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of + the actual struct is irrelevant. 5. We cannot easily + change it. Because this type is embedded in many locations, + updates to this type will affect numerous schemas. Don''t + make new APIs embed an underspecified API type they + do not control. Instead of using this type, create a + locally provided and used type that is well-focused + on your reference. For example, ServiceReferences for + admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + .' + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object + instead of an entire object, this string should + contain a valid JSON/Go field access statement, + such as desiredState.manifest.containers[2]. For + example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container + that triggered the event) or if no container name + is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part + of an object. TODO: this design is not final and + this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this + reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: array + workload: + description: 'ObjectReference contains enough information + to let you inspect or modify the referred object. --- + New uses of this type are discouraged because of difficulty + describing its usage when embedded in APIs. 1. Ignored + fields. It includes many fields which are not generally + honored. For instance, ResourceVersion and FieldPath + are both very rarely valid in actual usage. 2. Invalid + usage help. It is impossible to add specific help for + individual usage. In most embedded usages, there are + particular restrictions like, "must refer only to + types A and B" or "UID not honored" or "name must be restricted". Those + cannot be well described when embedded. 3. Inconsistent + validation. Because the usages are different, the validation + rules are different by usage, which makes it hard for + users to predict what will happen. 4. The fields are + both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during + interpretation and require a REST mapping. In most cases, + the dependency is on the group,resource tuple and + the version of the actual struct is irrelevant. 5. We + cannot easily change it. Because this type is embedded + in many locations, updates to this type will affect + numerous schemas. Don''t make new APIs embed an underspecified + API type they do not control. Instead of using this type, + create a locally provided and used type that is well-focused + on your reference. For example, ServiceReferences for + admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + .' + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a + valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container + that triggered the event) or if no container name + is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to + have some well-defined way of referencing a part of + an object. TODO: this design is not final and this + field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this + reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: object + type: array + type: object + type: array probe-interval: description: ProbeInterval is the amount of time in seconds between probing tries. @@ -120,6 +272,131 @@ spec: status: description: A HealthScopeStatus represents the observed state of a HealthScope. properties: + appHealthConditions: + description: AppHealthConditions represents health condition of applications + in the scope + items: + description: AppHealthCondition represents health condition of an + application + properties: + appName: + type: string + components: + items: + description: WorkloadHealthCondition represents informative + health condition of a workload. + properties: + componentName: + description: ComponentName represents the component name + if target is a workload + type: string + customStatusMsg: + type: string + diagnosis: + type: string + healthStatus: + description: HealthStatus represents health status strings. + type: string + targetWorkload: + description: 'ObjectReference contains enough information + to let you inspect or modify the referred object. --- + New uses of this type are discouraged because of difficulty + describing its usage when embedded in APIs. 1. Ignored + fields. It includes many fields which are not generally + honored. For instance, ResourceVersion and FieldPath + are both very rarely valid in actual usage. 2. Invalid + usage help. It is impossible to add specific help for + individual usage. In most embedded usages, there are + particular restrictions like, "must refer only to + types A and B" or "UID not honored" or "name must be restricted". Those + cannot be well described when embedded. 3. Inconsistent + validation. Because the usages are different, the validation + rules are different by usage, which makes it hard for + users to predict what will happen. 4. The fields are + both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during + interpretation and require a REST mapping. In most cases, + the dependency is on the group,resource tuple and + the version of the actual struct is irrelevant. 5. We + cannot easily change it. Because this type is embedded + in many locations, updates to this type will affect + numerous schemas. Don''t make new APIs embed an underspecified + API type they do not control. Instead of using this type, + create a locally provided and used type that is well-focused + on your reference. For example, ServiceReferences for + admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + .' + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a + valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container + that triggered the event) or if no container name + is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to + have some well-defined way of referencing a part of + an object. TODO: this design is not final and this + field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this + reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + traits: + items: + description: TraitHealthCondition represents informative + health condition of a trait. + properties: + customStatusMsg: + type: string + diagnosis: + type: string + healthStatus: + description: HealthStatus represents health status + strings. + type: string + resource: + type: string + type: + type: string + required: + - healthStatus + - resource + - type + type: object + type: array + workloadStatus: + description: WorkloadStatus represents status of workloads + whose HealthStatus is UNKNOWN. + type: string + required: + - healthStatus + type: object + type: array + required: + - appName + type: object + type: array conditions: description: Conditions of the resource. items: @@ -155,15 +432,18 @@ spec: type: array healthConditions: description: WorkloadHealthConditions represents health condition of - workloads in the scope + workloads in the scope Use AppHealthConditions to provide app level + status items: description: WorkloadHealthCondition represents informative health - condition. + condition of a workload. properties: componentName: description: ComponentName represents the component name if target is a workload type: string + customStatusMsg: + type: string diagnosis: type: string healthStatus: @@ -229,6 +509,28 @@ spec: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string type: object + traits: + items: + description: TraitHealthCondition represents informative health + condition of a trait. + properties: + customStatusMsg: + type: string + diagnosis: + type: string + healthStatus: + description: HealthStatus represents health status strings. + type: string + resource: + type: string + type: + type: string + required: + - healthStatus + - resource + - type + type: object + type: array workloadStatus: description: WorkloadStatus represents status of workloads whose HealthStatus is UNKNOWN. diff --git a/pkg/appfile/parser.go b/pkg/appfile/parser.go index a979bc228..1c6aa532d 100644 --- a/pkg/appfile/parser.go +++ b/pkg/appfile/parser.go @@ -182,7 +182,7 @@ func (p *Parser) parseWorkload(ctx context.Context, comp common.ApplicationCompo workload.Traits = append(workload.Traits, trait) } for scopeType, instanceName := range comp.Scopes { - gvk, err := getScopeGVK(ctx, p.client, p.dm, scopeType) + gvk, err := GetScopeGVK(ctx, p.client, p.dm, scopeType) if err != nil { return nil, err } @@ -338,7 +338,8 @@ func getComponentSetting(settingParamName string, params map[string]interface{}) return nil, fmt.Errorf("failed to get the value of component setting %s", settingParamName) } -func getScopeGVK(ctx context.Context, cli client.Reader, dm discoverymapper.DiscoveryMapper, +// GetScopeGVK get grouped API version of the given scope +func GetScopeGVK(ctx context.Context, cli client.Reader, dm discoverymapper.DiscoveryMapper, name string) (schema.GroupVersionKind, error) { var gvk schema.GroupVersionKind sd := new(v1alpha2.ScopeDefinition) diff --git a/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope.go b/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope.go index b4d532379..e1ea5765c 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope.go +++ b/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope.go @@ -34,7 +34,13 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + terraformtypes "github.com/oam-dev/terraform-controller/api/types" + terraformapi "github.com/oam-dev/terraform-controller/api/v1beta1" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" + oamtypes "github.com/oam-dev/kubevela/apis/types" + af "github.com/oam-dev/kubevela/pkg/appfile" + "github.com/oam-dev/kubevela/pkg/cue/process" "github.com/oam-dev/kubevela/pkg/oam" ) @@ -68,9 +74,15 @@ var ( kindDaemonSet = reflect.TypeOf(apps.DaemonSet{}).Name() ) -// WorkloadHealthCondition holds health status of any resource +// AppHealthCondition holds health status of an application +type AppHealthCondition = v1alpha2.AppHealthCondition + +// WorkloadHealthCondition holds health status of a workload type WorkloadHealthCondition = v1alpha2.WorkloadHealthCondition +// TraitHealthCondition holds health status of a trait +type TraitHealthCondition = v1alpha2.TraitHealthCondition + // ScopeHealthCondition holds health condition of a scope type ScopeHealthCondition = v1alpha2.ScopeHealthCondition @@ -429,3 +441,164 @@ func (p PeerHealthConditions) MergePeerWorkloadsConditions(basic *WorkloadHealth } } } + +// CUEBasedHealthCheck check workload and traits health through CUE-based health checking approach. +func CUEBasedHealthCheck(ctx context.Context, c client.Client, wlRef core.ObjectReference, ns string, appfile *af.Appfile) (*WorkloadHealthCondition, []*TraitHealthCondition) { + wlHealth := &WorkloadHealthCondition{ + TargetWorkload: wlRef, + } + + o := &unstructured.Unstructured{} + o.SetGroupVersionKind(wlRef.GroupVersionKind()) + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: wlRef.Name}, o); err != nil { + wlHealth.HealthStatus = StatusUnhealthy + wlHealth.Diagnosis = errors.Wrap(err, errHealthCheck).Error() + return wlHealth, nil + } + compName := getComponentNameFromLabel(o) + wlHealth.ComponentName = compName + + var wl *af.Workload + for _, v := range appfile.Workloads { + if v.Name == compName { + wl = v + break + } + } + if wl == nil { + // almost impossible + return nil, nil + } + + var pCtx process.Context + + // if error occurs when check workload health, it's not allowed to check traits + // because CUE-based health checking replies on valid process context + okToCheckTrait := false + + func() { + if wl.ConfigNotReady { + wlHealth.HealthStatus = StatusUnhealthy + wlHealth.Diagnosis = "secrets or configs not ready" + return + } + + var ( + outputSecretName string + err error + ) + if wl.IsSecretProducer() { + outputSecretName, err = af.GetOutputSecretNames(wl) + if err != nil { + wlHealth.HealthStatus = StatusUnhealthy + wlHealth.Diagnosis = errors.Wrap(err, errHealthCheck).Error() + return + } + } + switch wl.CapabilityCategory { + case oamtypes.TerraformCategory: + pCtx = af.NewBasicContext(wl, appfile.Name, appfile.RevisionName, appfile.Namespace) + pCtx.InsertSecrets(outputSecretName, wl.RequiredSecrets) + ctx := context.Background() + var configuration terraformapi.Configuration + if err := c.Get(ctx, client.ObjectKey{Name: wl.Name, Namespace: ns}, &configuration); err != nil { + wlHealth.HealthStatus = StatusUnhealthy + wlHealth.Diagnosis = errors.Wrap(err, errHealthCheck).Error() + } + if configuration.Status.State != terraformtypes.Available { + wlHealth.HealthStatus = StatusUnhealthy + } else { + wlHealth.HealthStatus = StatusHealthy + } + wlHealth.Diagnosis = configuration.Status.Message + okToCheckTrait = true + default: + pCtx = process.NewContext(ns, wl.Name, appfile.Name, appfile.RevisionName) + pCtx.InsertSecrets(outputSecretName, wl.RequiredSecrets) + if wl.CapabilityCategory != oamtypes.CUECategory { + templateStr, err := af.GenerateCUETemplate(wl) + if err != nil { + wlHealth.HealthStatus = StatusUnhealthy + wlHealth.Diagnosis = errors.Wrap(err, errHealthCheck).Error() + return + } + wl.FullTemplate.TemplateStr = templateStr + } + + if err := wl.EvalContext(pCtx); err != nil { + wlHealth.HealthStatus = StatusUnhealthy + wlHealth.Diagnosis = errors.Wrap(err, errHealthCheck).Error() + return + } + // if workload has no CUE-based health template, skip check workload, + // but still okay to check traits because process context is ready + if len(wl.FullTemplate.Health) == 0 { + wlHealth = nil + okToCheckTrait = true + return + } + isHealthy, err := wl.EvalHealth(pCtx, c, ns) + if err != nil { + wlHealth.HealthStatus = StatusUnhealthy + wlHealth.Diagnosis = errors.Wrap(err, errHealthCheck).Error() + return + } + if isHealthy { + wlHealth.HealthStatus = StatusHealthy + } else { + // TODO(wonderflow): we should add a custom way to let the template say why it's unhealthy, only a bool flag is not enough + wlHealth.HealthStatus = StatusUnhealthy + } + wlHealth.CustomStatusMsg, err = wl.EvalStatus(pCtx, c, ns) + if err != nil { + wlHealth.Diagnosis = errors.Wrap(err, errHealthCheck).Error() + } + okToCheckTrait = true + } + }() + + traits := make([]*v1alpha2.TraitHealthCondition, len(wl.Traits)) + for i, tr := range wl.Traits { + tHealth := &v1alpha2.TraitHealthCondition{ + Type: tr.Name, + } + if !okToCheckTrait { + tHealth.HealthStatus = StatusUnknown + tHealth.Diagnosis = "error occurs in checking workload health" + traits[i] = tHealth + continue + } + + if len(tr.FullTemplate.Health) == 0 { + tHealth.HealthStatus = StatusUnknown + tHealth.Diagnosis = "no CUE-based health check template" + traits[i] = tHealth + continue + } + if err := tr.EvalContext(pCtx); err != nil { + tHealth.HealthStatus = StatusUnhealthy + tHealth.Diagnosis = errors.Wrap(err, errHealthCheck).Error() + traits[i] = tHealth + continue + } + isHealthy, err := tr.EvalHealth(pCtx, c, ns) + if err != nil { + tHealth.HealthStatus = StatusUnhealthy + tHealth.Diagnosis = errors.Wrap(err, errHealthCheck).Error() + traits[i] = tHealth + continue + } + if isHealthy { + tHealth.HealthStatus = StatusHealthy + } else { + // TODO(wonderflow): we should add a custom way to let the template say why it's unhealthy, only a bool flag is not enough + tHealth.HealthStatus = StatusUnhealthy + } + tHealth.CustomStatusMsg, err = tr.EvalStatus(pCtx, c, ns) + if err != nil { + tHealth.Diagnosis = errors.Wrap(err, errHealthCheck).Error() + } + traits[i] = tHealth + } + return wlHealth, traits +} diff --git a/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope_controller.go b/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope_controller.go index 54017a3b3..a3fedacdd 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope_controller.go +++ b/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope_controller.go @@ -26,6 +26,7 @@ import ( "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" @@ -34,8 +35,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + af "github.com/oam-dev/kubevela/pkg/appfile" "github.com/oam-dev/kubevela/pkg/controller/common" controller "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev" + "github.com/oam-dev/kubevela/pkg/cue/packages" + "github.com/oam-dev/kubevela/pkg/oam" + "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" ) const ( @@ -54,20 +60,22 @@ const ( ) // Setup adds a controller that reconciles HealthScope. -func Setup(mgr ctrl.Manager, _ controller.Args) error { +func Setup(mgr ctrl.Manager, args controller.Args) error { name := "oam/" + strings.ToLower(v1alpha2.HealthScopeGroupKind) - + r := NewReconciler(mgr, WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name)))) + r.dm = args.DiscoveryMapper + r.pd = args.PackageDiscover return ctrl.NewControllerManagedBy(mgr). Named(name). For(&v1alpha2.HealthScope{}). - Complete(NewReconciler(mgr, - WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name))), - )) + Complete(r) } // A Reconciler reconciles OAM Scopes by keeping track of the health status of components. type Reconciler struct { client client.Client + dm discoverymapper.DiscoveryMapper + pd *packages.PackageDiscover record event.Recorder // traitChecker represents checker fetching health condition from HealthCheckTrait traitChecker WorloadHealthChecker @@ -151,26 +159,38 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco klog.InfoS("healthScope", "uid", hs.GetUID(), "version", hs.GetResourceVersion()) - scopeCondition, wlConditions := r.GetScopeHealthStatus(ctx, hs) + scopeCondition, appConditions := r.GetScopeHealthStatus(ctx, hs) klog.V(common.LogDebug).InfoS("Successfully ran health check", "scope", hs.Name) r.record.Event(hs, event.Normal(reasonHealthCheck, "Successfully ran health check")) elapsed := time.Since(start) hs.Status.ScopeHealthCondition = scopeCondition - hs.Status.WorkloadHealthConditions = wlConditions + hs.Status.AppHealthConditions = appConditions return reconcile.Result{RequeueAfter: interval - elapsed}, errors.Wrap(r.UpdateStatus(ctx, hs), errUpdateHealthScopeStatus) } // GetScopeHealthStatus get the status of the healthscope based on workload resources. -func (r *Reconciler) GetScopeHealthStatus(ctx context.Context, healthScope *v1alpha2.HealthScope) (ScopeHealthCondition, []*WorkloadHealthCondition) { +func (r *Reconciler) GetScopeHealthStatus(ctx context.Context, healthScope *v1alpha2.HealthScope) (ScopeHealthCondition, []*AppHealthCondition) { klog.InfoS("Get scope health status", "name", healthScope.GetName()) scopeCondition := ScopeHealthCondition{ HealthStatus: StatusHealthy, // if no workload referenced, scope is healthy by default } - scopeWLRefs := healthScope.Spec.WorkloadReferences - if len(scopeWLRefs) == 0 { - return scopeCondition, []*WorkloadHealthCondition{} + + var wlRefs []corev1.ObjectReference + if len(healthScope.Spec.WorkloadReferences) > 0 { + wlRefs = healthScope.Spec.WorkloadReferences + } else { + wlRefs = make([]corev1.ObjectReference, 0) + for _, app := range healthScope.Spec.AppRefs { + for _, comp := range app.CompReferences { + wlRefs = append(wlRefs, comp.Workload) + } + } + } + + if len(wlRefs) == 0 { + return scopeCondition, []*AppHealthCondition{} } timeout := defaultTimeout @@ -180,21 +200,46 @@ func (r *Reconciler) GetScopeHealthStatus(ctx context.Context, healthScope *v1al ctxWithTimeout, cancel := context.WithTimeout(ctx, timeout) defer cancel() - // process workloads concurrently - workloadHealthConditionsC := make(chan *WorkloadHealthCondition, len(scopeWLRefs)) - var wg sync.WaitGroup - wg.Add(len(scopeWLRefs)) + appfiles, appNames := r.CollectAppfilesAndAppNames(ctx, wlRefs, healthScope.GetNamespace()) - for _, workloadRef := range scopeWLRefs { + type wlHealthResult struct { + name string + w *WorkloadHealthCondition + } + // process workloads concurrently + wlHealthResultsC := make(chan wlHealthResult, len(wlRefs)) + var wg sync.WaitGroup + wg.Add(len(wlRefs)) + + for _, workloadRef := range wlRefs { go func(resRef corev1.ObjectReference) { defer wg.Done() - var wlHealthCondition *WorkloadHealthCondition + var ( + wlHealthCondition *WorkloadHealthCondition + traitConditions []*TraitHealthCondition + ) + + if appfile, ok := appfiles[resRef]; ok { + wlHealthCondition, traitConditions = CUEBasedHealthCheck(ctx, r.client, resRef, healthScope.GetNamespace(), appfile) + if wlHealthCondition != nil { + klog.V(common.LogDebug).InfoS("Get health condition from CUE-based health check", "workload", resRef, "healthCondition", wlHealthCondition) + wlHealthCondition.Traits = traitConditions + wlHealthResultsC <- wlHealthResult{ + name: appNames[resRef], + w: wlHealthCondition, + } + return + } + } wlHealthCondition = r.traitChecker.Check(ctx, r.client, resRef, healthScope.GetNamespace()) if wlHealthCondition != nil { klog.V(common.LogDebug).InfoS("Get health condition from health check trait ", "workload", resRef, "healthCondition", wlHealthCondition) - // get healthCondition from HealthCheckTrait - workloadHealthConditionsC <- wlHealthCondition + wlHealthCondition.Traits = traitConditions + wlHealthResultsC <- wlHealthResult{ + name: appNames[resRef], + w: wlHealthCondition, + } return } @@ -203,26 +248,34 @@ func (r *Reconciler) GetScopeHealthStatus(ctx context.Context, healthScope *v1al if wlHealthCondition != nil { klog.V(common.LogDebug).InfoS("Get health condition from built-in checker", "workload", resRef, "healthCondition", wlHealthCondition) // found matched checker and get health condition - workloadHealthConditionsC <- wlHealthCondition + wlHealthCondition.Traits = traitConditions + wlHealthResultsC <- wlHealthResult{ + name: appNames[resRef], + w: wlHealthCondition, + } return } } // handle unknown workload klog.V(common.LogDebug).InfoS("Gpkg/controller/core.oam.dev/v1alpha2/setup.go:42:69et unknown workload", "workload", resRef) - workloadHealthConditionsC <- r.unknownChecker.Check(ctx, r.client, resRef, healthScope.GetNamespace()) + wlHealthCondition = r.unknownChecker.Check(ctx, r.client, resRef, healthScope.GetNamespace()) + wlHealthCondition.Traits = traitConditions + wlHealthResultsC <- wlHealthResult{ + name: appNames[resRef], + w: wlHealthCondition, + } }(workloadRef) } go func() { wg.Wait() - close(workloadHealthConditionsC) + close(wlHealthResultsC) }() + appHealthConditions := make([]*AppHealthCondition, 0) var healthyCount, unhealthyCount, unknownCount int64 - workloadHealthConditions := []*WorkloadHealthCondition{} - for wlC := range workloadHealthConditionsC { - workloadHealthConditions = append(workloadHealthConditions, wlC) - switch wlC.HealthStatus { //nolint:exhaustive + for wlC := range wlHealthResultsC { + switch wlC.w.HealthStatus { //nolint:exhaustive case StatusHealthy: healthyCount++ case StatusUnhealthy: @@ -232,17 +285,69 @@ func (r *Reconciler) GetScopeHealthStatus(ctx context.Context, healthScope *v1al default: unknownCount++ } + appended := false + for _, a := range appHealthConditions { + if a.AppName == wlC.name { + a.Components = append(a.Components, wlC.w) + appended = true + break + } + } + if !appended { + appHealth := &AppHealthCondition{ + AppName: wlC.name, + Components: []*v1alpha2.WorkloadHealthCondition{wlC.w}, + } + appHealthConditions = append(appHealthConditions, appHealth) + } } if unhealthyCount > 0 || unknownCount > 0 { // ANY unhealthy or unknown worloads make the whole scope unhealthy scopeCondition.HealthStatus = StatusUnhealthy } - scopeCondition.Total = int64(len(scopeWLRefs)) + scopeCondition.Total = int64(len(wlRefs)) scopeCondition.HealthyWorkloads = healthyCount scopeCondition.UnhealthyWorkloads = unhealthyCount scopeCondition.UnknownWorkloads = unknownCount - return scopeCondition, workloadHealthConditions + return scopeCondition, appHealthConditions +} + +// CollectAppfilesAndAppNames retrieve appfiles and app names for CUEBasedHealthCheck +func (r *Reconciler) CollectAppfilesAndAppNames(ctx context.Context, refs []corev1.ObjectReference, ns string) (map[corev1.ObjectReference]*af.Appfile, map[corev1.ObjectReference]string) { + appfiles := map[corev1.ObjectReference]*af.Appfile{} + appNames := map[corev1.ObjectReference]string{} + + tmps := map[string]*af.Appfile{} + for _, ref := range refs { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(ref.GroupVersionKind()) + if err := r.client.Get(ctx, client.ObjectKey{Name: ref.Name, Namespace: ns}, u); err != nil { + // no need to check error in this function + // HealthCheckFn will handle all errors latter + continue + } + appName := u.GetLabels()[oam.LabelAppName] + if appfile, ok := tmps[appName]; ok { + appfiles[ref] = appfile + appNames[ref] = appName + continue + } + app := &v1beta1.Application{} + if err := r.client.Get(ctx, client.ObjectKey{Name: appName, Namespace: ns}, app); err != nil { + continue + } + appParser := af.NewApplicationParser(r.client, r.dm, r.pd) + appfile, err := appParser.GenerateAppFile(ctx, app) + if err != nil { + continue + } + tmps[appName] = appfile + + appfiles[ref] = appfile + appNames[ref] = appName + } + return appfiles, appNames } // UpdateStatus updates v1alpha2.HealthScope's Status with retry.RetryOnConflict diff --git a/test/e2e-test/health_scope_test.go b/test/e2e-test/health_scope_test.go index 5f2855813..5a4c83cf9 100644 --- a/test/e2e-test/health_scope_test.go +++ b/test/e2e-test/health_scope_test.go @@ -33,7 +33,9 @@ import ( "github.com/oam-dev/kubevela/apis/core.oam.dev/common" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/pkg/oam/util" + utilcommon "github.com/oam-dev/kubevela/pkg/utils/common" ) var ( @@ -42,46 +44,19 @@ var ( var _ = Describe("HealthScope", func() { ctx := context.Background() - namespace := "health-scope-test" + var namespace string trueVar := true falseVar := false - ns := corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - }, - } + var ns corev1.Namespace BeforeEach(func() { - logf.Log.Info("Start to run a test, clean up previous resources") - // delete the namespace with all its resources - Expect(k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground))). - Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) - logf.Log.Info("make sure all the resources are removed") - objectKey := client.ObjectKey{ - Name: namespace, + namespace = randomNamespaceName("health-scope-test") + ns = corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, } - res := &corev1.Namespace{} - Eventually( - // gomega has a bug that can't take nil as the actual input, so has to make it a func - func() error { - return k8sClient.Get(ctx, objectKey, res) - }, - time.Second*120, time.Millisecond*500).Should(&util.NotFoundMatcher{}) - // recreate it - Eventually( - func() error { - return k8sClient.Create(ctx, &ns) - }, - time.Second*3, time.Millisecond*300).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + Expect(k8sClient.Create(ctx, &ns)).Should(Succeed()) - }) - AfterEach(func() { - logf.Log.Info("Clean up resources") - // delete the namespace with all its resources - Expect(k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(BeNil()) - }) - - It("Test an application config with health scope", func() { - healthScopeName := "example-health-scope" // create health scope definition sd := v1alpha2.ScopeDefinition{ ObjectMeta: metav1.ObjectMeta{ @@ -98,7 +73,16 @@ var _ = Describe("HealthScope", func() { } logf.Log.Info("Creating health scope definition") Expect(k8sClient.Create(ctx, &sd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + }) + AfterEach(func() { + logf.Log.Info("Clean up resources") + Expect(k8sClient.DeleteAllOf(ctx, &v1alpha2.HealthScope{}, client.InNamespace(namespace))).Should(BeNil()) + // delete the namespace with all its resources + Expect(k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(BeNil()) + }) + It("Test an application config with health scope", func() { + healthScopeName := "example-health-scope" // create health scope. hs := v1alpha2.HealthScope{ ObjectMeta: metav1.ObjectMeta{ @@ -110,8 +94,8 @@ var _ = Describe("HealthScope", func() { WorkloadReferences: []corev1.ObjectReference{}, }, } - logf.Log.Info("Creating health scope") Expect(k8sClient.Create(ctx, &hs)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + By("Check empty health scope is healthy") Eventually(func() v1alpha2.HealthStatus { k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: healthScopeName}, &hs) @@ -332,4 +316,103 @@ var _ = Describe("HealthScope", func() { HealthyWorkloads: int64(2), })) }) + + It("Test an application with health scope", func() { + By("Apply a healthy application") + var newApp v1beta1.Application + Expect(utilcommon.ReadYamlToObject("testdata/app/app_healthscope.yaml", &newApp)).Should(BeNil()) + newApp.Namespace = namespace + Eventually(func() error { + return k8sClient.Create(ctx, newApp.DeepCopy()) + }, 10*time.Second, 500*time.Millisecond).Should(Succeed()) + + By("Get Application latest status") + Eventually( + func() *common.Revision { + var app v1beta1.Application + k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: newApp.Name}, &app) + if app.Status.LatestRevision != nil { + return app.Status.LatestRevision + } + return nil + }, + time.Second*30, time.Millisecond*500).ShouldNot(BeNil()) + + By("Apply an unhealthy application") + newApp = v1beta1.Application{} + Expect(utilcommon.ReadYamlToObject("testdata/app/app_healthscope_unhealthy.yaml", &newApp)).Should(BeNil()) + newApp.Namespace = namespace + Eventually(func() error { + return k8sClient.Create(ctx, newApp.DeepCopy()) + }, 10*time.Second, 500*time.Millisecond).Should(Succeed()) + + By("Get Application latest status") + Eventually( + func() *common.Revision { + var app v1beta1.Application + k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: newApp.Name}, &app) + if app.Status.LatestRevision != nil { + return app.Status.LatestRevision + } + return nil + }, + time.Second*30, time.Millisecond*500).ShouldNot(BeNil()) + + By("Create HealthScope instance") + healthScopeName := "example-health-scope" + hs := v1alpha2.HealthScope{ + ObjectMeta: metav1.ObjectMeta{ + Name: healthScopeName, + Namespace: namespace, + }, + Spec: v1alpha2.HealthScopeSpec{ + ProbeTimeout: &varInt32_60, + WorkloadReferences: []corev1.ObjectReference{}, + }, + } + + // TODO(roywang) we haven't implemnet associating app references to + // health scope in app controller, so manually add app references now + hs.Spec.AppRefs = []v1alpha2.AppReference{{ + AppName: "app-healthscope", + CompReferences: []v1alpha2.CompReference{{ + CompName: "my-server", + Workload: corev1.ObjectReference{ + Kind: "Deployment", + Name: "my-server", + APIVersion: "apps/v1", + }, + }}, + }, { + AppName: "app-healthscope-unhealthy", + CompReferences: []v1alpha2.CompReference{{ + CompName: "my-server-unhealthy", + Workload: corev1.ObjectReference{ + Kind: "Deployment", + Name: "my-server-unhealthy", + APIVersion: "apps/v1", + }, + }}, + }} + Expect(k8sClient.Create(ctx, &hs)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + + By("Verify health scope") + healthScopeObject := client.ObjectKey{ + Name: healthScopeName, + Namespace: namespace, + } + healthScope := &v1alpha2.HealthScope{} + Eventually( + func() v1alpha2.ScopeHealthCondition { + *healthScope = v1alpha2.HealthScope{} + k8sClient.Get(ctx, healthScopeObject, healthScope) + return healthScope.Status.ScopeHealthCondition + }, + time.Second*60, time.Millisecond*500).Should(Equal(v1alpha2.ScopeHealthCondition{ + HealthStatus: v1alpha2.StatusUnhealthy, + Total: int64(2), + HealthyWorkloads: int64(1), + UnhealthyWorkloads: int64(1), + })) + }) }) diff --git a/test/e2e-test/testdata/app/app_healthscope.yaml b/test/e2e-test/testdata/app/app_healthscope.yaml new file mode 100644 index 000000000..10b2cd197 --- /dev/null +++ b/test/e2e-test/testdata/app/app_healthscope.yaml @@ -0,0 +1,20 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: app-healthscope +spec: + components: + - name: my-server + type: webservice + properties: + cmd: + - node + - server.js + image: oamdev/testapp:v1 + port: 8080 + traits: + - type: ingress + properties: + domain: test.my.domain + http: + "/": 8080 diff --git a/test/e2e-test/testdata/app/app_healthscope_unhealthy.yaml b/test/e2e-test/testdata/app/app_healthscope_unhealthy.yaml new file mode 100644 index 000000000..b8f01cbfb --- /dev/null +++ b/test/e2e-test/testdata/app/app_healthscope_unhealthy.yaml @@ -0,0 +1,14 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: app-healthscope-unhealthy +spec: + components: + - name: my-server-unhealthy + type: webservice + properties: + cmd: + - node + - server.js + image: oamdev/testapp:boom # make it unhealthy + port: 8080