Merge pull request #329 from captainroy-hy/track-comp-status

track status changing in vela init
This commit is contained in:
Jianbo Sun
2020-10-09 19:12:35 +08:00
committed by GitHub
8 changed files with 510 additions and 112 deletions
+1
View File
@@ -75,6 +75,7 @@ docker-push:
e2e-setup:
ginkgo version
ginkgo -v -r e2e/setup
kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=300s
bin/vela dashboard &
e2e-test:
+4 -1
View File
@@ -13,6 +13,7 @@ var (
workloadType = "webservice"
applicationName = "app-basic"
traitAlias = "scale"
appNameForInit = "initmyapp"
)
var _ = ginkgo.Describe("Application", func() {
@@ -26,6 +27,8 @@ var _ = ginkgo.Describe("Application", func() {
e2e.TraitManualScalerAttachContext("vela attach scale trait", traitAlias, applicationName)
e2e.ApplicationShowContext("app show", applicationName, workloadType)
e2e.ApplicationStatusContext("app status", applicationName, workloadType)
e2e.ApplicationCompStatusContext("comp status", applicationName, workloadType)
e2e.ApplicationCompStatusContext("comp status", applicationName, workloadType, envName)
e2e.ApplicationInitIntercativeCliContext("init", appNameForInit, workloadType)
e2e.WorkloadDeleteContext("delete", applicationName)
e2e.WorkloadDeleteContext("delete", appNameForInit)
})
+66
View File
@@ -7,9 +7,17 @@ import (
"strings"
"time"
"github.com/Netflix/go-expect"
"github.com/hinshun/vt10x"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
"github.com/onsi/gomega/gexec"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
oamcore "github.com/crossplane/oam-kubernetes-runtime/apis/core"
k8sruntime "k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
)
var rudrPath = GetCliBinary()
@@ -30,6 +38,16 @@ func Exec(cli string) (string, error) {
return string(s.Out.Contents()) + string(s.Err.Contents()), nil
}
func LongTimeExec(cli string, timeout time.Duration) (string, error) {
var output []byte
session, err := AsyncExec(cli)
if err != nil {
return string(output), err
}
s := session.Wait(timeout)
return string(s.Out.Contents()) + string(s.Err.Contents()), nil
}
func AsyncExec(cli string) (*gexec.Session, error) {
c := strings.Fields(cli)
commandName := path.Join(rudrPath, c[0])
@@ -38,9 +56,57 @@ func AsyncExec(cli string) (*gexec.Session, error) {
return session, err
}
func InteractiveExec(cli string, consoleFn func(*expect.Console)) (string, error) {
var output []byte
console, _, err := vt10x.NewVT10XConsole(expect.WithStdout(ginkgo.GinkgoWriter))
gomega.Expect(err).NotTo(gomega.HaveOccurred())
defer console.Close()
doneC := make(chan struct{})
go func() {
defer ginkgo.GinkgoRecover()
defer close(doneC)
consoleFn(console)
}()
c := strings.Fields(cli)
commandName := path.Join(rudrPath, c[0])
command := exec.Command(commandName, c[1:]...)
command.Stdin = console.Tty()
session, err := gexec.Start(command, console.Tty(), console.Tty())
s := session.Wait(30 * time.Second)
console.Tty().Close()
<-doneC
if err != nil {
return string(output), err
}
return string(s.Out.Contents()) + string(s.Err.Contents()), nil
}
func BeforeSuit() {
_, err := Exec("vela install")
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
//Without this line, will hit issue like `<string>: Error: unknown command "scale" for "vela"`
_, _ = Exec("vela system update")
}
func newK8sClient() (client.Client, error) {
conf, err := config.GetConfig()
if err != nil {
return nil, err
}
scheme := k8sruntime.NewScheme()
if err := clientgoscheme.AddToScheme(scheme); err != nil {
return nil, err
}
if err := oamcore.AddToScheme(scheme); err != nil {
return nil, err
}
k8sclient, err := client.New(conf, client.Options{Scheme: scheme})
if err != nil {
return nil, err
}
return k8sclient, nil
}
+70 -4
View File
@@ -1,14 +1,19 @@
package e2e
import (
ctx "context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
"time"
"github.com/Netflix/go-expect"
corev1alpha2 "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2"
"github.com/oam-dev/kubevela/pkg/server/apis"
"github.com/oam-dev/kubevela/pkg/server/util"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
@@ -186,13 +191,24 @@ var (
})
}
ApplicationCompStatusContext = func(context string, applicationName string, workloadType string) bool {
ApplicationCompStatusContext = func(context string, applicationName, workloadType, envName string) bool {
return ginkgo.Context(context, func() {
ginkgo.It("should get status for the component", func() {
cli := fmt.Sprintf("vela comp status %s", applicationName)
output, err := Exec(cli)
ginkgo.By("init new k8s client")
k8sclient, err := newK8sClient()
gomega.Expect(err).NotTo(gomega.HaveOccurred())
gomega.Expect(output).To(gomega.ContainSubstring(applicationName))
ginkgo.By("check AppConfig reconciled ready")
gomega.Eventually(func() int {
appConfig := &corev1alpha2.ApplicationConfiguration{}
_ = k8sclient.Get(ctx.Background(), client.ObjectKey{Name: applicationName, Namespace: "default"}, appConfig)
return len(appConfig.Status.Workloads)
}, 120*time.Second, 1*time.Second).ShouldNot(gomega.Equal(0))
cli := fmt.Sprintf("vela comp status %s", applicationName)
output, err := LongTimeExec(cli, 120*time.Second)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
gomega.Expect(output).To(gomega.ContainSubstring("Checking health status"))
// TODO(zzxwill) need to check workloadType after app status is refined
})
})
@@ -211,6 +227,56 @@ var (
})
}
ApplicationInitIntercativeCliContext = func(context string, appName string, workloadType string) bool {
return ginkgo.Context(context, func() {
ginkgo.It("should init app through interactive questions", func() {
cli := "vela init"
output, err := InteractiveExec(cli, func(c *expect.Console) {
data := []struct {
q, a string
}{
{
q: "Do you want to setup a domain for web service: ",
a: "testdomain",
},
{
q: "Provide an email for production certification: ",
a: "test@mail",
},
{
q: "What would you like to name your application: ",
a: appName,
},
{
q: "webservice",
a: workloadType,
},
{
q: "What would you name this webservice: ",
a: "mysvc",
},
{
q: "specify app image ",
a: "nginx:latest",
},
{
q: "specify port for container ",
a: "8080",
},
}
for _, qa := range data {
_, err := c.ExpectString(qa.q)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
_, err = c.SendLine(qa.a)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
}
_, _ = c.ExpectEOF()
})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
gomega.Expect(output).To(gomega.ContainSubstring("Initializing"))
})
})
}
// APIEnvInitContext used for test api env
APIEnvInitContext = func(context string, envMeta apis.Environment) bool {
return ginkgo.Context("Post /envs/", func() {
+5 -1
View File
@@ -5,9 +5,11 @@ go 1.13
require (
cuelang.org/go v0.2.2
github.com/AlecAivazis/survey/v2 v2.1.1
github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8
github.com/briandowns/spinner v1.11.1
github.com/coreos/prometheus-operator v0.41.1
github.com/crossplane/crossplane-runtime v0.9.0
github.com/crossplane/oam-kubernetes-runtime v0.1.1-0.20200909070723-78b84f2c4799
github.com/crossplane/oam-kubernetes-runtime v0.2.1
github.com/fatih/color v1.9.0
github.com/gertd/go-pluralize v0.1.7
github.com/ghodss/yaml v1.0.0
@@ -17,7 +19,9 @@ require (
github.com/google/go-cmp v0.5.2
github.com/google/go-github/v32 v32.1.0
github.com/gosuri/uitable v0.0.4
github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174
github.com/jetstack/cert-manager v0.14.3
github.com/kyokomi/emoji v2.2.4+incompatible
github.com/mholt/archiver/v3 v3.3.0
github.com/oam-dev/trait-injector v0.0.0-20200331033130-0a27b176ffc4
github.com/onsi/ginkgo v1.13.0
+6 -2
View File
@@ -197,6 +197,8 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2
github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
github.com/brancz/kube-rbac-proxy v0.5.0/go.mod h1:cL2VjiIFGS90Cjh5ZZ8+It6tMcBt8rwvuw2J6Mamnl0=
github.com/briandowns/spinner v1.11.1 h1:OixPqDEcX3juo5AjQZAnFPbeUA0jvkp2qzB5gOZJ/L0=
github.com/briandowns/spinner v1.11.1/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
github.com/bshuster-repo/logrus-logstash-hook v0.4.1 h1:pgAtgj+A31JBVtEHu2uHuEx0n+2ukqUJnS2vVe5pQNA=
github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng=
@@ -282,8 +284,8 @@ github.com/crossplane/crossplane-runtime v0.8.0/go.mod h1:gNY/21MLBaz5KNP7hmfXbB
github.com/crossplane/crossplane-runtime v0.9.0 h1:K6/tLhXKzhsEUUddTvEWWnQLLrawWyw1ptNK7NBDpDU=
github.com/crossplane/crossplane-runtime v0.9.0/go.mod h1:gNY/21MLBaz5KNP7hmfXbBXp8reYRbwY5B/97Kp4tgM=
github.com/crossplane/crossplane-tools v0.0.0-20200219001116-bb8b2ce46330/go.mod h1:C735A9X0x0lR8iGVOOxb49Mt70Ua4EM2b7PGaRPBLd4=
github.com/crossplane/oam-kubernetes-runtime v0.1.1-0.20200909070723-78b84f2c4799 h1:424LLFb7C8Qvy3wFZZ7HzmawlCeF32PNRTXXK5rKOk0=
github.com/crossplane/oam-kubernetes-runtime v0.1.1-0.20200909070723-78b84f2c4799/go.mod h1:UZ4eXkl/e4lKrAhK81Pz1sR90wqeuE9PgdwVXr8kDgI=
github.com/crossplane/oam-kubernetes-runtime v0.2.1 h1:C0kiSo9Tza/T+OhnjrP1yrQL+huPGxlINK7xK9VCYYo=
github.com/crossplane/oam-kubernetes-runtime v0.2.1/go.mod h1:D+MDS5vrJZWEA5cxr5kyzCSRQwrt1hLD3ONgC7sVMmc=
github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
github.com/daixiang0/gci v0.0.0-20200727065011-66f1df783cb2/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4=
@@ -888,6 +890,8 @@ github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LE
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8=
github.com/kyokomi/emoji v2.2.4+incompatible h1:np0woGKwx9LiHAQmwZx79Oc0rHpNw3o+3evou4BEPv4=
github.com/kyokomi/emoji v2.2.4+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
+18 -1
View File
@@ -70,7 +70,24 @@ func NewInitCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command {
if err != nil {
return err
}
o.IOStreams.Info("App Deployed Succeed")
ctx := context.Background()
initStatus, err := printTrackingInitStatus(ctx, o.client, o.IOStreams, o.workloadName, o.appName, o.Env)
if err != nil {
return err
}
if initStatus != compStatusInitialized {
return nil
}
deployStatus, err := printTrackingDeployStatus(ctx, o.client, o.IOStreams, o.workloadName, o.appName, o.Env)
if err != nil {
return err
}
if deployStatus != compStatusDeployed {
return nil
}
//TODO(wonderflow) Wait for app running, and print trait info such as route, domain
return printComponentStatus(context.Background(), o.client, o.IOStreams, o.workloadName, o.appName, o.Env)
},
+340 -103
View File
@@ -8,13 +8,19 @@ import (
"strings"
"time"
"github.com/briandowns/spinner"
runtimev1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1"
"github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2"
"github.com/fatih/color"
"github.com/ghodss/yaml"
"github.com/gosuri/uitable"
"github.com/kyokomi/emoji"
"github.com/pkg/errors"
"github.com/spf13/cobra"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/duration"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/oam-dev/kubevela/api/types"
@@ -26,17 +32,17 @@ import (
type HealthStatus = v1alpha2.HealthStatus
const (
// StatusNotDiagnosed means there's no health check info returned from the scope.
StatusNotDiagnosed HealthStatus = "NOT DIAGNOSED"
// HealthStatusNotDiagnosed means there's no health scope refered or unknown health status returned
HealthStatusNotDiagnosed HealthStatus = "NOT DIAGNOSED"
)
const (
// StatusHealthy represents healthy status.
StatusHealthy = v1alpha2.StatusHealthy
// StatusUnhealthy represents unhealthy status.
StatusUnhealthy = v1alpha2.StatusUnhealthy
// StatusUnknown represents unknown status.
StatusUnknown = v1alpha2.StatusUnknown
// HealthStatusHealthy represents healthy status.
HealthStatusHealthy = v1alpha2.StatusHealthy
// HealthStatusUnhealthy represents unhealthy status.
HealthStatusUnhealthy = v1alpha2.StatusUnhealthy
// HealthStatusUnknown represents unknown status.
HealthStatusUnknown = v1alpha2.StatusUnknown
)
// WorkloadHealthCondition holds health status of any resource
@@ -45,6 +51,30 @@ type WorkloadHealthCondition = v1alpha2.WorkloadHealthCondition
// ScopeHealthCondition holds health condition of a scope
type ScopeHealthCondition = v1alpha2.ScopeHealthCondition
var (
kindHealthScope = reflect.TypeOf(v1alpha2.HealthScope{}).Name()
)
// CompStatus represents the status of a component during "vela init"
type CompStatus int
const (
compStatusInitializing CompStatus = iota
compStatusInitFail
compStatusInitialized
compStatusDeploying
compStatusDeployFail
compStatusDeployed
compStatusHealthChecking
compStatusHealthCheckDone
compStatusUnknown
)
const (
ErrNotLoadAppConfig = "cannot load the application"
ErrFmtNotInitialized = "oam-core-controller cannot initilize the component: %s"
)
const (
firstElemPrefix = `├─`
lastElemPrefix = `└─`
@@ -61,7 +91,16 @@ var (
)
var (
kindHealthScope = reflect.TypeOf(v1alpha2.HealthScope{}).Name()
emojiSucceed = emoji.Sprint(":check_mark_button:")
emojiFail = emoji.Sprint(":cross_mark:")
emojiTimeout = emoji.Sprint(":heavy_exclamation_mark:")
)
const (
trackingInterval time.Duration = 1 * time.Second
initTimeout time.Duration = 30 * time.Second
deployTimeout time.Duration = 30 * time.Second
healthCheckBufferTime time.Duration = 120 * time.Second
)
func NewAppStatusCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command {
@@ -150,7 +189,6 @@ func printAppStatus(ctx context.Context, c client.Client, ioStreams cmdutil.IOSt
// map componentName <=> WorkloadHealthCondition
func getWorkloadHealthConditions(ctx context.Context, c client.Client, app *application.Application, ns string) (map[string]*WorkloadHealthCondition, error) {
hs := &v1alpha2.HealthScope{}
// only use default health scope
hsName := application.FormatDefaultHealthScopeName(app.Name)
@@ -169,7 +207,7 @@ func getWorkloadHealthConditions(ctx context.Context, c client.Client, app *appl
}
if r[compName] == nil {
r[compName] = &WorkloadHealthCondition{
HealthStatus: StatusNotDiagnosed,
HealthStatus: HealthStatusNotDiagnosed,
}
}
}
@@ -212,94 +250,59 @@ func NewCompStatusCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Comm
}
func printComponentStatus(ctx context.Context, c client.Client, ioStreams cmdutil.IOStreams, compName, appName string, env *types.EnvMeta) error {
ioStreams.Infof("Showing status of Component %s deployed in Environment %s\n", compName, env.Name)
var app *application.Application
var err error
if appName != "" {
app, err = application.Load(env.Name, appName)
} else {
app, err = application.MatchAppByComp(env.Name, compName)
}
app, appConfig, err := getApp(ctx, c, compName, appName, env)
if err != nil {
return err
}
var appConfig v1alpha2.ApplicationConfiguration
if err = c.Get(ctx, client.ObjectKey{Namespace: env.Namespace, Name: app.Name}, &appConfig); err != nil {
return err
if app == nil || appConfig == nil {
return errors.New(ErrNotLoadAppConfig)
}
var wlStatus v1alpha2.WorkloadStatus // very important
for _, v := range appConfig.Status.Workloads {
if v.ComponentName == compName {
wlStatus = v
break
wlStatus, foundWlStatus := getWorkloadStatusFromAppConfig(appConfig, compName)
if !foundWlStatus {
appConfigReconcileStatus := appConfig.Status.GetCondition(runtimev1alpha1.TypeSynced).Status
switch appConfigReconcileStatus {
case corev1.ConditionUnknown:
ioStreams.Info("\nUnknown error occurs during component initialization. \nPlease check OAM controller ...")
case corev1.ConditionTrue:
ioStreams.Info("\nThe component is still under initialization, please try again later ...")
case corev1.ConditionFalse:
appConfigConditionMsg := appConfig.Status.GetCondition(runtimev1alpha1.TypeSynced).Message
ioStreams.Info("\nError occurs in OAM runtime during component initialization.")
ioStreams.Infof("\nOAM controller condition message: %s \n", appConfigConditionMsg)
}
}
if wlStatus.ComponentName == "" {
//TODO(roywang) cannot get workload instance
//TODO(roywang) if appConfig reconcile condition is false, then output err msg
ioStreams.Infof("\nComponent is not created yet.")
return nil
}
var (
healthColor *color.Color
healthStatus HealthStatus
healthInfo string
workloadType string
)
var healthInfo string
var healthStatus HealthStatus
workloadType = wlStatus.Reference.Kind
sHealthCheck := newTrackingSpinner("Checking health status ...")
sHealthCheck.Start()
// check whether referenced a HealthScope
var healthScopeName string
for _, v := range wlStatus.Scopes {
if v.Reference.Kind == kindHealthScope {
healthScopeName = v.Reference.Name
}
}
if len(healthScopeName) == 0 {
// no health scope referenced
healthStatus = StatusNotDiagnosed
statusInfo, err := getWorkloadStatus(ctx, c, env.Namespace, wlStatus.Reference)
HealthCheckLoop:
for {
time.Sleep(trackingInterval)
var healthcheckStatus CompStatus
healthcheckStatus, healthStatus, healthInfo, err = trackHealthCheckingStatus(ctx, c, compName, appName, env)
if err != nil {
sHealthCheck.Stop()
ioStreams.Info(red.Sprintf("Health checking failed!"))
return err
}
// format output
statusInfo = strings.ReplaceAll(statusInfo, "\n", "\n\t")
healthInfo = fmt.Sprintf("%s \n\n\tWARN: The component is not in any HealthScope. \n%s", healthStatus, statusInfo)
} else {
var healthScope v1alpha2.HealthScope
if err = c.Get(ctx, client.ObjectKey{Namespace: env.Namespace, Name: healthScopeName}, &healthScope); err != nil {
return err
}
var wlhc *v1alpha2.WorkloadHealthCondition
for _, v := range healthScope.Status.WorkloadHealthConditions {
if v.ComponentName == compName {
wlhc = v
}
}
healthStatus = wlhc.HealthStatus
if healthStatus == StatusUnknown {
healthStatus = StatusNotDiagnosed
statusInfo, err := getWorkloadStatus(ctx, c, env.Namespace, wlStatus.Reference)
if err != nil {
return err
}
// format output
statusInfo = strings.ReplaceAll(statusInfo, "\n", "\n\t")
healthInfo = fmt.Sprintf("%s \n\n\tWARN: The component type is unknown to HealthScope. \n\t%s", healthStatus, statusInfo)
} else {
healthInfo = fmt.Sprintf("%s %s", healthStatus, wlhc.Diagnosis)
if healthcheckStatus == compStatusHealthCheckDone {
sHealthCheck.Stop()
break HealthCheckLoop
}
}
ioStreams.Infof("Showing status of Component %s deployed in Environment %s\n", compName, env.Name)
ioStreams.Infof(white.Sprint("Component Status:\n"))
healthColor = getHealthStatusColor(healthStatus)
ioStreams.Infof("\tName: %s %s(type) %s \n", compName, workloadType, healthColor.Sprint(healthInfo))
workloadType := wlStatus.Reference.Kind
healthColor := getHealthStatusColor(healthStatus)
healthInfo = strings.ReplaceAll(healthInfo, "\n", "\n\t") // formart healthInfo output
ioStreams.Infof("\tName: %s %s(type) %s %s\n",
compName, workloadType, healthColor.Sprint(healthStatus), healthColor.Sprint(healthInfo))
traits, err := app.GetTraits(compName)
if err != nil {
@@ -339,6 +342,258 @@ func printComponentStatus(ctx context.Context, c client.Client, ioStreams cmduti
return nil
}
func getWorkloadInstanceStatusAndCreationTime(ctx context.Context, c client.Client, ns string, wlRef runtimev1alpha1.TypedReference) (string, bool, metav1.Time, error) {
wlUnstruct := unstructured.Unstructured{}
wlUnstruct.SetGroupVersionKind(wlRef.GroupVersionKind())
if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: wlRef.Name},
&wlUnstruct); err != nil {
return "", false, metav1.Time{}, err
}
ct := wlUnstruct.GetCreationTimestamp()
statusData, foundStatus, _ := unstructured.NestedMap(wlUnstruct.Object, "status")
if foundStatus {
statusYaml, err := yaml.Marshal(statusData)
if err != nil {
return "", false, ct, err
}
return string(statusYaml), true, ct, nil
}
return "", false, ct, nil
}
func printTrackingInitStatus(ctx context.Context, c client.Client, ioStreams cmdutil.IOStreams, compName, appName string, env *types.EnvMeta) (CompStatus, error) {
tInit := time.Now()
sInit := newTrackingSpinner("Initializing ...")
sInit.Start()
TrackInitLoop:
for {
time.Sleep(trackingInterval)
if time.Since(tInit) > initTimeout {
ioStreams.Info(red.Sprintf("\n%sInitialization Timeout After %s!",
emojiTimeout, duration.HumanDuration(time.Since(tInit))))
ioStreams.Info(red.Sprint("Please make sure oam-core-controller is installed."))
sInit.Stop()
return compStatusUnknown, nil
}
initStatus, failMsg, err := trackInitializeStatus(ctx, c, compName, appName, env)
if err != nil {
return compStatusUnknown, err
}
switch initStatus {
case compStatusInitializing:
continue
case compStatusInitialized:
ioStreams.Info(green.Sprintf("\n%sInitialization Succeed!", emojiSucceed))
sInit.Stop()
break TrackInitLoop
case compStatusInitFail:
ioStreams.Info(red.Sprintf("\n%sInitialization Failed!", emojiFail))
ioStreams.Info(red.Sprintf("Reason: %s", failMsg))
sInit.Stop()
return compStatusInitFail, nil
}
}
return compStatusInitialized, nil
}
func trackInitializeStatus(ctx context.Context, c client.Client, compName, appName string, env *types.EnvMeta) (CompStatus, string, error) {
app, appConfig, err := getApp(ctx, c, compName, appName, env)
if err != nil {
return compStatusUnknown, "", err
}
if app == nil || appConfig == nil {
return compStatusUnknown, "", errors.New(ErrNotLoadAppConfig)
}
_, foundWlStatus := getWorkloadStatusFromAppConfig(appConfig, compName)
appConfigReconcileStatus := appConfig.Status.GetCondition(runtimev1alpha1.TypeSynced).Status
switch appConfigReconcileStatus {
case corev1.ConditionUnknown:
return compStatusInitializing, "", nil
case corev1.ConditionTrue:
if foundWlStatus {
return compStatusInitialized, "", nil
}
return compStatusInitializing, "", nil
case corev1.ConditionFalse:
appConfigConditionMsg := appConfig.Status.GetCondition(runtimev1alpha1.TypeSynced).Message
return compStatusInitFail, appConfigConditionMsg, nil
}
return compStatusInitializing, "", nil
}
func printTrackingDeployStatus(ctx context.Context, c client.Client, ioStreams cmdutil.IOStreams, compName, appName string, env *types.EnvMeta) (CompStatus, error) {
sDeploy := newTrackingSpinner("Deploying ...")
sDeploy.Start()
TrackDeployLoop:
for {
time.Sleep(trackingInterval)
deployStatus, failMsg, err := trackDeployStatus(ctx, c, compName, appName, env)
if err != nil {
return compStatusUnknown, err
}
switch deployStatus {
case compStatusDeploying:
continue
case compStatusDeployed:
ioStreams.Info(green.Sprintf("\n%sDeployment Succeed!", emojiSucceed))
sDeploy.Stop()
break TrackDeployLoop
case compStatusDeployFail:
ioStreams.Info(red.Sprintf("\n%sDeployment Failed!", emojiFail))
ioStreams.Info(red.Sprintf("Reason: %s", failMsg))
sDeploy.Stop()
return compStatusDeployFail, nil
}
}
return compStatusDeployed, nil
}
func trackDeployStatus(ctx context.Context, c client.Client, compName, appName string, env *types.EnvMeta) (CompStatus, string, error) {
app, appConfig, err := getApp(ctx, c, compName, appName, env)
if err != nil {
return compStatusUnknown, "", err
}
if app == nil || appConfig == nil {
return compStatusUnknown, "", errors.New(ErrNotLoadAppConfig)
}
wlStatus, foundWlStatus := getWorkloadStatusFromAppConfig(appConfig, compName)
// make sure component already initilized
if !foundWlStatus {
appConfigConditionMsg := appConfig.Status.GetCondition(runtimev1alpha1.TypeSynced).Message
return compStatusUnknown, "", fmt.Errorf(ErrFmtNotInitialized, appConfigConditionMsg)
}
wlRef := wlStatus.Reference
//TODO(roywang) temporarily use status to judge workload controller is running
// even not every workload has `status` field
//TODO(roywang) check whether traits are ready
_, foundStatus, ct, err := getWorkloadInstanceStatusAndCreationTime(ctx, c, env.Namespace, wlRef)
if err != nil {
return compStatusUnknown, "", err
}
if foundStatus {
return compStatusDeployed, "", nil
}
// if not found workload status in AppConfig
// then use age to check whether the worload controller is running
if time.Since(ct.Time) > deployTimeout {
return compStatusDeployFail, fmt.Sprintf("The controller of [%s] is not installed or running.",
wlStatus.Reference.GroupVersionKind().String()), nil
}
return compStatusDeploying, "", nil
}
func trackHealthCheckingStatus(ctx context.Context, c client.Client, compName, appName string, env *types.EnvMeta) (CompStatus, HealthStatus, string, error) {
app, appConfig, err := getApp(ctx, c, compName, appName, env)
if err != nil {
return compStatusUnknown, HealthStatusNotDiagnosed, "", err
}
if app == nil || appConfig == nil {
return compStatusUnknown, HealthStatusNotDiagnosed, "", errors.New(ErrNotLoadAppConfig)
}
wlStatus, foundWlStatus := getWorkloadStatusFromAppConfig(appConfig, compName)
// make sure component already initilized
if !foundWlStatus {
appConfigConditionMsg := appConfig.Status.GetCondition(runtimev1alpha1.TypeSynced).Message
return compStatusUnknown, HealthStatusUnknown, "", fmt.Errorf(ErrFmtNotInitialized, appConfigConditionMsg)
}
// check whether referenced a HealthScope
var healthScopeName string
for _, v := range wlStatus.Scopes {
if v.Reference.Kind == kindHealthScope {
healthScopeName = v.Reference.Name
}
}
if len(healthScopeName) == 0 {
// no health scope referenced
statusInfo, _, _, err := getWorkloadInstanceStatusAndCreationTime(ctx, c, env.Namespace, wlStatus.Reference)
if err != nil {
return compStatusUnknown, HealthStatusUnknown, "", err
}
return compStatusHealthCheckDone, HealthStatusNotDiagnosed, statusInfo, nil
}
var healthScope v1alpha2.HealthScope
if err = c.Get(ctx, client.ObjectKey{Namespace: env.Namespace, Name: healthScopeName}, &healthScope); err != nil {
return compStatusUnknown, HealthStatusUnknown, "", err
}
var wlhc *v1alpha2.WorkloadHealthCondition
for _, v := range healthScope.Status.WorkloadHealthConditions {
if v.ComponentName == compName {
wlhc = v
}
}
if wlhc == nil {
return compStatusUnknown, HealthStatusUnknown, "", fmt.Errorf("cannot get health condition from the health scope: %s", healthScope.Name)
}
healthStatus := wlhc.HealthStatus
if healthStatus == HealthStatusUnknown {
healthStatus = HealthStatusNotDiagnosed
statusInfo, _, _, err := getWorkloadInstanceStatusAndCreationTime(ctx, c, env.Namespace, wlStatus.Reference)
if err != nil {
return compStatusUnknown, HealthStatusUnknown, "", errors.Wrap(err, "WARN: The component type is unknown to HealthScope and cannot get status.")
}
healthInfo := fmt.Sprintf("WARN: The component type is unknown to HealthScope.\nYou may check component status with [%s/%s] status: \n%s",
wlhc.TargetWorkload.Kind, wlhc.TargetWorkload.Name, statusInfo)
return compStatusHealthCheckDone, healthStatus, healthInfo, nil
}
if healthStatus == HealthStatusUnhealthy {
cTime := appConfig.GetCreationTimestamp()
if time.Since(cTime.Time) <= healthCheckBufferTime {
return compStatusHealthChecking, HealthStatusUnknown, "", nil
}
}
return compStatusHealthCheckDone, healthStatus, wlhc.Diagnosis, nil
}
func getApp(ctx context.Context, c client.Client, compName, appName string, env *types.EnvMeta) (*application.Application, *v1alpha2.ApplicationConfiguration, error) {
var app *application.Application
var err error
if appName != "" {
app, err = application.Load(env.Name, appName)
} else {
app, err = application.MatchAppByComp(env.Name, compName)
}
if err != nil {
return nil, nil, err
}
appConfig := &v1alpha2.ApplicationConfiguration{}
if err = c.Get(ctx, client.ObjectKey{Namespace: env.Namespace, Name: app.Name}, appConfig); err != nil {
return nil, nil, err
}
return app, appConfig, nil
}
func getWorkloadStatusFromAppConfig(appConfig *v1alpha2.ApplicationConfiguration, compName string) (v1alpha2.WorkloadStatus, bool) {
foundWlStatus := false
wlStatus := v1alpha2.WorkloadStatus{}
if appConfig == nil {
return wlStatus, foundWlStatus
}
for _, v := range appConfig.Status.Workloads {
if v.ComponentName == compName {
wlStatus = v
foundWlStatus = true
break
}
}
return wlStatus, foundWlStatus
}
func newTrackingSpinner(suffix string) *spinner.Spinner {
suffixColor := color.New(color.Bold, color.FgGreen)
return spinner.New(
spinner.CharSets[14],
100*time.Millisecond,
spinner.WithColor("green"),
spinner.WithHiddenCursor(true),
spinner.WithSuffix(suffixColor.Sprintf(" %s", suffix)))
}
func printPrefix(p string) string {
if strings.HasSuffix(p, firstElemPrefix) {
p = strings.Replace(p, firstElemPrefix, pipe, strings.Count(p, firstElemPrefix)-1)
@@ -357,34 +612,16 @@ func printPrefix(p string) string {
func getHealthStatusColor(s HealthStatus) *color.Color {
var c *color.Color
switch s {
case StatusHealthy:
case HealthStatusHealthy:
c = green
case StatusUnhealthy:
case HealthStatusUnhealthy:
c = red
case StatusUnknown:
case HealthStatusUnknown:
c = yellow
case StatusNotDiagnosed:
case HealthStatusNotDiagnosed:
c = yellow
default:
c = red
}
return c
}
func getWorkloadStatus(ctx context.Context, c client.Client, ns string, wlRef runtimev1alpha1.TypedReference) (string, error) {
wlUnstruct := unstructured.Unstructured{}
wlUnstruct.SetGroupVersionKind(wlRef.GroupVersionKind())
if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: wlRef.Name},
&wlUnstruct); err != nil {
return "", err
}
statusData, foundStatus, _ := unstructured.NestedMap(wlUnstruct.Object, "status")
if foundStatus {
statusYaml, err := yaml.Marshal(statusData)
if err != nil {
return "", err
}
return string(statusYaml), nil
}
return red.Sprintf("Error: Cannot get status info. \nPlease check the controller of workload: %s.", wlRef.GroupVersionKind().String()), nil
}