mirror of
https://github.com/kubevela/kubevela.git
synced 2026-02-14 18:10:21 +00:00
429 lines
14 KiB
Go
429 lines
14 KiB
Go
/*
|
|
Copyright 2021 The KubeVela Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/gosuri/uitable"
|
|
"github.com/oam-dev/cluster-gateway/pkg/generated/clientset/versioned"
|
|
"github.com/pkg/errors"
|
|
"github.com/spf13/cobra"
|
|
v1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/labels"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/rest"
|
|
apiregistrationV1beta "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
|
|
apiregistration "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1beta1"
|
|
metrics "k8s.io/metrics/pkg/client/clientset/versioned"
|
|
"sigs.k8s.io/yaml"
|
|
|
|
"github.com/oam-dev/kubevela/apis/types"
|
|
"github.com/oam-dev/kubevela/pkg/multicluster"
|
|
"github.com/oam-dev/kubevela/pkg/utils/common"
|
|
)
|
|
|
|
const (
|
|
// FlagSpecify specifies the deployment name
|
|
FlagSpecify = "specify"
|
|
// FlagOutputFormat specifies the output format. One of: (wide | yaml)
|
|
FlagOutputFormat = "output"
|
|
// APIServiceName is the name of APIService
|
|
APIServiceName = "v1alpha1.cluster.core.oam.dev"
|
|
// UnknownMetric represent that we can't compute the metric data
|
|
UnknownMetric = "N/A"
|
|
)
|
|
|
|
// NewSystemCommand print system detail info
|
|
func NewSystemCommand(c common.Args, order string) *cobra.Command {
|
|
cmd := &cobra.Command{
|
|
Use: "system",
|
|
Short: "Manage system.",
|
|
Long: "Manage system, including printing the system deployment information in vela-system namespace and diagnosing the system's health.",
|
|
Example: "# Check all deployments information in all namespaces with label app.kubernetes.io/name=vela-core :\n" +
|
|
"> vela system info\n" +
|
|
"# Specify a deployment name with a namespace to check detail information:\n" +
|
|
"> vela system info -s kubevela-vela-core -n vela-system\n" +
|
|
"# Diagnose the system's health:\n" +
|
|
"> vela system diagnose\n",
|
|
Annotations: map[string]string{
|
|
types.TagCommandType: types.TypeSystem,
|
|
types.TagCommandOrder: order,
|
|
},
|
|
}
|
|
cmd.AddCommand(
|
|
NewSystemInfoCommand(c),
|
|
NewSystemDiagnoseCommand(c))
|
|
return cmd
|
|
}
|
|
|
|
// NewSystemInfoCommand prints system detail info
|
|
func NewSystemInfoCommand(c common.Args) *cobra.Command {
|
|
cmd := &cobra.Command{
|
|
Use: "info",
|
|
Short: "Print the system deployment detail information in all namespaces with label app.kubernetes.io/name=vela-core.",
|
|
Long: "Print the system deployment detail information in all namespaces with label app.kubernetes.io/name=vela-core.",
|
|
Args: cobra.ExactArgs(0),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
// Get deploymentName from flag
|
|
deployName, err := cmd.Flags().GetString(FlagSpecify)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to get deployment name flag")
|
|
}
|
|
// Get output format from flag
|
|
outputFormat, err := cmd.Flags().GetString(FlagOutputFormat)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to get output format flag")
|
|
}
|
|
if outputFormat != "" {
|
|
outputFormatOptions := map[string]struct{}{
|
|
"wide": {},
|
|
"yaml": {},
|
|
}
|
|
if _, exist := outputFormatOptions[outputFormat]; !exist {
|
|
return errors.Errorf("Outputformat must in wide | yaml !")
|
|
}
|
|
}
|
|
// Get kube config
|
|
if outputFormat != "" {
|
|
outputFormatOptions := map[string]struct{}{
|
|
"wide": {},
|
|
"yaml": {},
|
|
}
|
|
if _, exist := outputFormatOptions[outputFormat]; !exist {
|
|
return errors.Errorf("Outputformat must in wide | yaml !")
|
|
}
|
|
}
|
|
// Get kube config
|
|
config, err := c.GetConfig()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Get clientset
|
|
clientset, err := kubernetes.NewForConfig(config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
// Get deploymentsClient in all namespace
|
|
deployments, err := clientset.AppsV1().Deployments(metav1.NamespaceAll).List(
|
|
ctx,
|
|
metav1.ListOptions{
|
|
LabelSelector: "app.kubernetes.io/name=vela-core",
|
|
},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if deployName != "" {
|
|
// DeployName is not empty, print the specified deployment's information
|
|
found := false
|
|
for _, deployment := range deployments.Items {
|
|
if deployment.Name == deployName {
|
|
table := SpecifiedFormatPrinter(deployment)
|
|
cmd.Println(table.String())
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return errors.Errorf("deployment \"%s\" not found", deployName)
|
|
}
|
|
} else {
|
|
// Get metrics clientset
|
|
mc, err := metrics.NewForConfig(config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch outputFormat {
|
|
case "":
|
|
table, err := NormalFormatPrinter(ctx, deployments, mc)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cmd.Println(table.String())
|
|
case "wide":
|
|
table, err := WideFormatPrinter(ctx, deployments, mc)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cmd.Println(table.String())
|
|
case "yaml":
|
|
str, err := YamlFormatPrinter(deployments)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cmd.Println(str)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
Annotations: map[string]string{
|
|
types.TagCommandType: types.TypeSystem,
|
|
},
|
|
}
|
|
cmd.Flags().StringP(FlagSpecify, "s", "", "Specify the name of the deployment to check detail information. If empty, it will print all deployments information. Default to be empty.")
|
|
cmd.Flags().StringP(FlagOutputFormat, "o", "", "Specifies the output format. One of: (wide | yaml)")
|
|
return cmd
|
|
}
|
|
|
|
// SpecifiedFormatPrinter prints the specified deployment's information
|
|
func SpecifiedFormatPrinter(deployment v1.Deployment) *uitable.Table {
|
|
table := newUITable().
|
|
AddRow("Name:", deployment.Name).
|
|
AddRow("Namespace:", deployment.Namespace).
|
|
AddRow("CreationTimestamp:", deployment.CreationTimestamp).
|
|
AddRow("Labels:", Map2Str(deployment.Labels)).
|
|
AddRow("Annotations:", Map2Str(deployment.Annotations)).
|
|
AddRow("Selector:", Map2Str(deployment.Spec.Selector.MatchLabels)).
|
|
AddRow("Image:", deployment.Spec.Template.Spec.Containers[0].Image).
|
|
AddRow("Args:", strings.Join(deployment.Spec.Template.Spec.Containers[0].Args, "\n")).
|
|
AddRow("Envs:", GetEnvVariable(deployment.Spec.Template.Spec.Containers[0].Env)).
|
|
AddRow("Limits:", CPUMem(deployment.Spec.Template.Spec.Containers[0].Resources.Limits)).
|
|
AddRow("Requests:", CPUMem(deployment.Spec.Template.Spec.Containers[0].Resources.Requests))
|
|
table.MaxColWidth = 120
|
|
return table
|
|
}
|
|
|
|
// CPUMem returns the upsage of cpu and memory
|
|
func CPUMem(resourceList corev1.ResourceList) string {
|
|
b := new(bytes.Buffer)
|
|
fmt.Fprintf(b, "cpu=%s\n", resourceList.Cpu())
|
|
fmt.Fprintf(b, "memory=%s", resourceList.Memory())
|
|
return b.String()
|
|
}
|
|
|
|
// Map2Str converts map to string
|
|
func Map2Str(m map[string]string) string {
|
|
b := new(bytes.Buffer)
|
|
for key, value := range m {
|
|
fmt.Fprintf(b, "%s=%s\n", key, value)
|
|
}
|
|
if len(b.String()) > 1 {
|
|
return b.String()[:len(b.String())-1]
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// NormalFormatPrinter prints information in format of normal
|
|
func NormalFormatPrinter(ctx context.Context, deployments *v1.DeploymentList, mc *metrics.Clientset) (*uitable.Table, error) {
|
|
table := newUITable().AddRow("NAME", "NAMESPACE", "READY PODS", "IMAGE", "CPU(cores)", "MEMORY(bytes)")
|
|
cpuMetricMap, memMetricMap := ComputeMetricByDeploymentName(ctx, deployments, mc)
|
|
for _, deploy := range deployments.Items {
|
|
table.AddRow(
|
|
deploy.Name,
|
|
deploy.Namespace,
|
|
fmt.Sprintf("%d/%d", deploy.Status.ReadyReplicas, deploy.Status.Replicas),
|
|
deploy.Spec.Template.Spec.Containers[0].Image,
|
|
cpuMetricMap[deploy.Name],
|
|
memMetricMap[deploy.Name],
|
|
)
|
|
}
|
|
return table, nil
|
|
}
|
|
|
|
// WideFormatPrinter prints information in format of wide
|
|
func WideFormatPrinter(ctx context.Context, deployments *v1.DeploymentList, mc *metrics.Clientset) (*uitable.Table, error) {
|
|
table := newUITable().AddRow("NAME", "NAMESPACE", "READY PODS", "IMAGE", "CPU(cores)", "MEMORY(bytes)", "ARGS", "ENVS")
|
|
table.MaxColWidth = 100
|
|
cpuMetricMap, memMetricMap := ComputeMetricByDeploymentName(ctx, deployments, mc)
|
|
for _, deploy := range deployments.Items {
|
|
table.AddRow(
|
|
deploy.Name,
|
|
deploy.Namespace,
|
|
fmt.Sprintf("%d/%d", deploy.Status.ReadyReplicas, deploy.Status.Replicas),
|
|
deploy.Spec.Template.Spec.Containers[0].Image,
|
|
cpuMetricMap[deploy.Name],
|
|
memMetricMap[deploy.Name],
|
|
strings.Join(deploy.Spec.Template.Spec.Containers[0].Args, " "),
|
|
limitStringLength(GetEnvVariable(deploy.Spec.Template.Spec.Containers[0].Env), 180),
|
|
)
|
|
}
|
|
return table, nil
|
|
}
|
|
|
|
// YamlFormatPrinter prints information in format of yaml
|
|
func YamlFormatPrinter(deployments *v1.DeploymentList) (string, error) {
|
|
str := ""
|
|
for _, deployment := range deployments.Items {
|
|
// Set ManagedFields to nil because it's too long to read
|
|
deployment.ManagedFields = nil
|
|
deploymentYaml, err := yaml.Marshal(deployment)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
str += string(deploymentYaml)
|
|
}
|
|
return str, nil
|
|
}
|
|
|
|
// ComputeMetricByDeploymentName computes cpu and memory metric of deployment
|
|
func ComputeMetricByDeploymentName(ctx context.Context, deployments *v1.DeploymentList, mc *metrics.Clientset) (cpuMetricMap, memMetricMap map[string]string) {
|
|
cpuMetricMap = make(map[string]string)
|
|
memMetricMap = make(map[string]string)
|
|
podMetricsList, err := mc.MetricsV1beta1().PodMetricses(metav1.NamespaceAll).List(
|
|
ctx,
|
|
metav1.ListOptions{},
|
|
)
|
|
if err != nil {
|
|
for _, deploy := range deployments.Items {
|
|
cpuMetricMap[deploy.Name] = UnknownMetric
|
|
memMetricMap[deploy.Name] = UnknownMetric
|
|
}
|
|
return
|
|
}
|
|
|
|
for _, deploy := range deployments.Items {
|
|
cpuUsage, memUsage := int64(0), int64(0)
|
|
for _, pod := range podMetricsList.Items {
|
|
if strings.HasPrefix(pod.Name, deploy.Name) {
|
|
for _, container := range pod.Containers {
|
|
cpuUsage += container.Usage.Cpu().MilliValue()
|
|
memUsage += container.Usage.Memory().Value() / (1024 * 1024)
|
|
}
|
|
}
|
|
}
|
|
cpuMetricMap[deploy.Name] = fmt.Sprintf("%dm", cpuUsage)
|
|
memMetricMap[deploy.Name] = fmt.Sprintf("%dMi", memUsage)
|
|
}
|
|
return
|
|
}
|
|
|
|
// GetEnvVariable gets the environment variables
|
|
func GetEnvVariable(envList []corev1.EnvVar) (envStr string) {
|
|
for _, env := range envList {
|
|
envStr += fmt.Sprintf("%s=%s ", env.Name, env.Value)
|
|
}
|
|
if len(envStr) == 0 {
|
|
return "-"
|
|
}
|
|
return
|
|
}
|
|
|
|
// NewSystemDiagnoseCommand create command to help user to diagnose system's health
|
|
func NewSystemDiagnoseCommand(c common.Args) *cobra.Command {
|
|
cmd := &cobra.Command{
|
|
Use: "diagnose",
|
|
Short: "Diagnoses system problems.",
|
|
Long: "Diagnoses system problems.",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
// Diagnose clusters' health
|
|
fmt.Println("------------------------------------------------------")
|
|
fmt.Println("Diagnosing health of clusters...")
|
|
k8sClient, err := c.GetClient()
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to get k8s client")
|
|
}
|
|
clusters, err := multicluster.ListVirtualClusters(context.Background(), k8sClient)
|
|
if err != nil {
|
|
return errors.Wrap(err, "fail to get registered cluster")
|
|
}
|
|
// Get kube config
|
|
config, err := c.GetConfig()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, cluster := range clusters {
|
|
clusterName := cluster.Name
|
|
if clusterName == multicluster.ClusterLocalName {
|
|
continue
|
|
}
|
|
content, err := versioned.NewForConfigOrDie(config).ClusterV1alpha1().ClusterGateways().RESTClient(clusterName).Get().AbsPath("healthz").DoRaw(context.TODO())
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed connect cluster %s", clusterName)
|
|
}
|
|
cmd.Printf("Connect to cluster %s successfully.\n%s\n", clusterName, string(content))
|
|
}
|
|
fmt.Println("Result: Clusters are fine~")
|
|
fmt.Println("------------------------------------------------------")
|
|
// Diagnoses the link of hub APIServer to cluster-gateway
|
|
fmt.Println("------------------------------------------------------")
|
|
fmt.Println("Diagnosing the link of hub APIServer to cluster-gateway...")
|
|
// Get clientset
|
|
clientset, err := apiregistration.NewForConfig(config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
apiService, err := clientset.APIServices().Get(ctx, APIServiceName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, condition := range apiService.Status.Conditions {
|
|
if condition.Type == "Available" {
|
|
if condition.Status != "True" {
|
|
cmd.Printf("APIService \"%s\" is not available! \nMessage: %s\n", APIServiceName, condition.Message)
|
|
return CheckAPIService(ctx, config, apiService)
|
|
}
|
|
cmd.Printf("APIService \"%s\" is available!\n", APIServiceName)
|
|
}
|
|
}
|
|
fmt.Println("Result: The link of hub APIServer to cluster-gateway is fine~")
|
|
fmt.Println("------------------------------------------------------")
|
|
// Todo: Diagnose others
|
|
return nil
|
|
},
|
|
Annotations: map[string]string{
|
|
types.TagCommandType: types.TypeSystem,
|
|
},
|
|
}
|
|
return cmd
|
|
}
|
|
|
|
// CheckAPIService checks the APIService
|
|
func CheckAPIService(ctx context.Context, config *rest.Config, apiService *apiregistrationV1beta.APIService) error {
|
|
svcName := apiService.Spec.Service.Name
|
|
svcNamespace := apiService.Spec.Service.Namespace
|
|
clientset, err := kubernetes.NewForConfig(config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
svc, err := clientset.CoreV1().Services(svcNamespace).Get(ctx, svcName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
set := labels.Set(svc.Spec.Selector)
|
|
listOptions := metav1.ListOptions{LabelSelector: set.AsSelector().String()}
|
|
pods, err := clientset.CoreV1().Pods(svcNamespace).List(ctx, listOptions)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(pods.Items) == 0 {
|
|
return errors.Errorf("No available pods in %s namespace with label %s.", svcNamespace, set.AsSelector().String())
|
|
}
|
|
for _, pod := range pods.Items {
|
|
for _, status := range pod.Status.ContainerStatuses {
|
|
if !status.Ready {
|
|
for _, condition := range pod.Status.Conditions {
|
|
if condition.Status != "True" {
|
|
return errors.Errorf("Pod %s is not ready. Condition \"%s\" status: %s.", pod.Name, condition.Type, condition.Status)
|
|
}
|
|
}
|
|
return errors.Errorf("Pod %s is not ready.", pod.Name)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|