diff --git a/references/cli/status.go b/references/cli/status.go index 63995835c..842555302 100644 --- a/references/cli/status.go +++ b/references/cli/status.go @@ -24,6 +24,8 @@ import ( "strings" "time" + "k8s.io/client-go/rest" + "github.com/fatih/color" "github.com/olekukonko/tablewriter" "github.com/pkg/errors" @@ -53,6 +55,7 @@ import ( cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" types2 "github.com/oam-dev/kubevela/pkg/velaql/providers/query/types" "github.com/oam-dev/kubevela/references/appfile" + references "github.com/oam-dev/kubevela/references/common" ) // HealthStatus represents health status strings. @@ -118,7 +121,10 @@ func NewAppStatusCommand(c common.Args, order string, ioStreams cmdutil.IOStream vela status first-vela-app -o yaml # Get raw Application status using jsonpath - vela status first-vela-app -o jsonpath='{.status}'`, + vela status first-vela-app -o jsonpath='{.status}' + + # Get Application metrics status + vela status first-vela-app --metrics`, RunE: func(cmd *cobra.Command, args []string) error { // check args argsLength := len(args) @@ -163,6 +169,16 @@ func NewAppStatusCommand(c common.Args, order string, ioStreams cmdutil.IOStream } return printAppEndpoints(ctx, appName, namespace, f, c, false) } + + restConf, err := c.GetConfig() + if err != nil { + return err + } + + if showMetrics, err := cmd.Flags().GetBool("metrics"); showMetrics && err == nil { + return printMetrics(newClient, restConf, appName, namespace) + } + if outputFormat != "" { return printRawApplication(context.Background(), c, outputFormat, cmd.OutOrStdout(), namespace, appName) } @@ -182,6 +198,7 @@ func NewAppStatusCommand(c common.Args, order string, ioStreams cmdutil.IOStream cmd.Flags().BoolVarP(&detail, "detail", "d", false, "display more details in the application like input/output data in context. Note that if you want to show the realtime details of application resources, please use it with --tree") cmd.Flags().StringP("detail-format", "", "inline", "the format for displaying details, must be used with --detail. Can be one of inline, wide, list, table, raw.") cmd.Flags().StringVarP(&outputFormat, "output", "o", "", "raw Application output format. One of: (json, yaml, jsonpath)") + cmd.Flags().BoolP("metrics", "m", false, "show resource quota and consumption metrics of the application") addNamespaceAndEnvArg(cmd) return cmd } @@ -555,3 +572,33 @@ func printRawApplication(ctx context.Context, c common.Args, format string, out _, err = out.Write([]byte(str)) return err } + +// printMetrics prints the resource num and resource metrics of an application +func printMetrics(c client.Client, conf *rest.Config, appName, appNamespace string) error { + app := new(v1beta1.Application) + err := c.Get(context.Background(), client.ObjectKey{ + Name: appName, + Namespace: appNamespace, + }, app) + if err != nil { + return err + } + metrics, err := references.LoadApplicationMetrics(c, conf, app) + if err != nil { + return err + } + fmt.Println() + fmt.Printf("Kubernetes Resources created:\n") + fmt.Printf(" * Number of Pods: %d\n", metrics.Resource.PodNum) + fmt.Printf(" * Number of Containers: %d\n", metrics.Resource.ContainerNum) + fmt.Printf(" * Number of Managed Resource: %d\n", metrics.Resource.SubresourceNum) + fmt.Printf(" * Number of Nodes: %d\n", metrics.Resource.NodeNum) + fmt.Printf(" * Number of Clusters: %d\n", metrics.Resource.ClusterNum) + fmt.Println() + fmt.Printf("Underlying Physical Resoures consumed:\n") + fmt.Printf(" * Total CPU(cores): %d m\n", metrics.Status.CPU) + fmt.Printf(" * Total MEMORY(bytes): %d Mi\n", metrics.Status.Memory) + fmt.Printf(" * Total Storage(bytes): %d Gi\n", metrics.Status.Storage) + fmt.Println() + return nil +} diff --git a/references/cli/top/model/managed_resource.go b/references/cli/top/model/managed_resource.go index 17789b938..aca84354d 100644 --- a/references/cli/top/model/managed_resource.go +++ b/references/cli/top/model/managed_resource.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/oam-dev/kubevela/pkg/velaql/providers/query" + "github.com/oam-dev/kubevela/references/common" ) // ManagedResource is managed resource of application @@ -42,14 +43,7 @@ type ManagedResourceList []ManagedResource func ListManagedResource(ctx context.Context, c client.Client) (ManagedResourceList, error) { name := ctx.Value(&CtxKeyAppName).(string) namespace := ctx.Value(&CtxKeyNamespace).(string) - opt := query.Option{ - Name: name, - Namespace: namespace, - Filter: query.FilterOption{}, - } - - collector := query.NewAppCollector(c, opt) - appResList, err := collector.CollectResourceFromApp(context.Background()) + appResList, err := common.ListApplicationResource(c, name, namespace) if err != nil { return ManagedResourceList{}, err } diff --git a/references/cli/top/model/pod.go b/references/cli/top/model/pod.go index 3671864a3..cc34f8136 100644 --- a/references/cli/top/model/pod.go +++ b/references/cli/top/model/pod.go @@ -29,6 +29,7 @@ import ( "github.com/oam-dev/kubevela/pkg/velaql/providers/query" "github.com/oam-dev/kubevela/references/cli/top/utils" + "github.com/oam-dev/kubevela/references/common" ) // Pod represent the k8s pod resource instance @@ -72,7 +73,7 @@ func ListPods(ctx context.Context, cfg *rest.Config, c client.Client) (PodList, }, WithTree: true, } - resource, err := collectResource(ctx, c, opt) + resource, err := common.CollectApplicationResource(ctx, c, opt) if err != nil { return PodList{}, err diff --git a/references/cli/top/model/suit_test.go b/references/cli/top/model/suit_test.go index d2cdc0a89..9c23f2a82 100644 --- a/references/cli/top/model/suit_test.go +++ b/references/cli/top/model/suit_test.go @@ -71,6 +71,14 @@ var _ = BeforeSuite(func(done Done) { k8sClient, err = client.New(cfg, client.Options{Scheme: common.Scheme}) Expect(err).Should(BeNil()) Expect(k8sClient).ToNot(BeNil()) + + // create namespace + By("create namespace") + _ = k8sClient.Create(context.Background(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + }) // create app name, namespace := "first-vela-app", "default" testApp := &v1beta1.Application{ diff --git a/references/common/application.go b/references/common/application.go index a78b4f02f..c133ffe52 100644 --- a/references/common/application.go +++ b/references/common/application.go @@ -25,6 +25,7 @@ import ( "github.com/fatih/color" terraformapi "github.com/oam-dev/terraform-controller/api/v1beta2" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/serializer/json" apitypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -37,6 +38,8 @@ import ( "github.com/oam-dev/kubevela/pkg/utils/apply" "github.com/oam-dev/kubevela/pkg/utils/common" cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" + "github.com/oam-dev/kubevela/pkg/velaql/providers/query" + querytypes "github.com/oam-dev/kubevela/pkg/velaql/providers/query/types" "github.com/oam-dev/kubevela/references/appfile" "github.com/oam-dev/kubevela/references/appfile/api" "github.com/oam-dev/kubevela/references/appfile/template" @@ -265,3 +268,45 @@ func ApplyApplication(app corev1beta1.Application, ioStream cmdutil.IOStreams, c ioStream.Infof(Info(&app)) return nil } + +// CollectApplicationResource collects all resources of an application +func CollectApplicationResource(ctx context.Context, c client.Client, opt query.Option) ([]unstructured.Unstructured, error) { + app := new(corev1beta1.Application) + appKey := client.ObjectKey{Name: opt.Name, Namespace: opt.Namespace} + if err := c.Get(context.Background(), appKey, app); err != nil { + return nil, err + } + collector := query.NewAppCollector(c, opt) + appResList, err := collector.ListApplicationResources(context.Background(), app) + if err != nil { + return nil, err + } + var resources = make([]unstructured.Unstructured, 0) + for _, res := range appResList { + if res.ResourceTree != nil { + resources = append(resources, sonLeafResource(*res, res.ResourceTree, opt.Filter.Kind, opt.Filter.APIVersion)...) + } + if (opt.Filter.Kind == "" && opt.Filter.APIVersion == "") || (res.Kind == opt.Filter.Kind && res.APIVersion == opt.Filter.APIVersion) { + var object unstructured.Unstructured + object.SetAPIVersion(opt.Filter.APIVersion) + object.SetKind(opt.Filter.Kind) + if err := c.Get(ctx, apitypes.NamespacedName{Namespace: res.Namespace, Name: res.Name}, &object); err == nil { + resources = append(resources, object) + } + } + } + return resources, nil +} + +func sonLeafResource(res querytypes.AppliedResource, node *querytypes.ResourceTreeNode, kind string, apiVersion string) []unstructured.Unstructured { + objects := make([]unstructured.Unstructured, 0) + if node.LeafNodes != nil { + for i := 0; i < len(node.LeafNodes); i++ { + objects = append(objects, sonLeafResource(res, node.LeafNodes[i], kind, apiVersion)...) + } + } + if (kind == "" && apiVersion == "") || (node.Kind == kind && node.APIVersion == apiVersion) { + objects = append(objects, node.Object) + } + return objects +} diff --git a/references/common/metrics.go b/references/common/metrics.go new file mode 100644 index 000000000..02daa538b --- /dev/null +++ b/references/common/metrics.go @@ -0,0 +1,193 @@ +/* +Copyright 2022 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "context" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/metrics/pkg/apis/metrics/v1beta1" + metricsclientset "k8s.io/metrics/pkg/client/clientset/versioned" + "sigs.k8s.io/controller-runtime/pkg/client" + + appv1beta1 "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/pkg/velaql/providers/query" +) + +// ApplicationMetrics is the metrics of application +type ApplicationMetrics struct { + Status *ApplicationMetricsStatus + Resource *ApplicationResourceStatus +} + +// ApplicationMetricsStatus is the status of application metrics +type ApplicationMetricsStatus struct { + CPU uint64 + Memory uint64 + Storage uint64 +} + +// ApplicationResourceStatus is the status of application resource +type ApplicationResourceStatus struct { + NodeNum int + ClusterNum int + SubresourceNum int + PodNum int + ContainerNum int +} + +// GetPodMetrics get pod metrics +func GetPodMetrics(metricsClient metricsclientset.Interface, pod v1.Pod, allNamespaces bool) (*v1beta1.PodMetrics, error) { + ns := metav1.NamespaceAll + if !allNamespaces { + ns = pod.Namespace + } + m, err := metricsClient.MetricsV1beta1().PodMetricses(ns).Get(context.Background(), pod.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return m, nil +} + +// GetPodStorage get pod storage +func GetPodStorage(client client.Client, pod v1.Pod) (storages []v1.PersistentVolumeClaim) { + for _, v := range pod.Spec.Volumes { + if v.PersistentVolumeClaim != nil { + storage := v1.PersistentVolumeClaim{} + err := client.Get(context.Background(), types.NamespacedName{Name: v.PersistentVolumeClaim.ClaimName, Namespace: pod.Namespace}, &storage) + if err != nil { + continue + } + storages = append(storages, storage) + } + } + return +} + +// ListApplicationResource list application resource +func ListApplicationResource(c client.Client, name, namespace string) ([]query.Resource, error) { + opt := query.Option{ + Name: name, + Namespace: namespace, + Filter: query.FilterOption{}, + } + + collector := query.NewAppCollector(c, opt) + appResList, err := collector.CollectResourceFromApp(context.Background()) + if err != nil { + return []query.Resource{}, err + } + return appResList, err +} + +// ListApplicationPods list application pods +func ListApplicationPods(c client.Client, app *appv1beta1.Application, components []string) []v1.Pod { + pods := make([]v1.Pod, 0) + opt := query.Option{ + Name: app.Name, + Namespace: app.Namespace, + Filter: query.FilterOption{ + Components: components, + APIVersion: "v1", + Kind: "Pod", + }, + WithTree: true, + } + + objects, err := CollectApplicationResource(context.Background(), c, opt) + if err != nil { + return pods + } + for _, object := range objects { + pod := v1.Pod{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &pod) + if err != nil { + continue + } + pods = append(pods, pod) + } + return pods +} + +// LoadApplicationMetrics load application metrics +func (appMetrics *ApplicationMetricsStatus) LoadApplicationMetrics(c client.Client, conf *rest.Config, pods []v1.Pod) { + metricsClient := metricsclientset.NewForConfigOrDie(conf) + cpu, memory, stroage := uint64(0), uint64(0), uint64(0) + + for _, pod := range pods { + podMetrics, err := GetPodMetrics(metricsClient, pod, false) + if err != nil { + continue + } + + for _, containerMetrics := range podMetrics.Containers { + cpu += uint64(containerMetrics.Usage.Cpu().MilliValue()) + memory += uint64(containerMetrics.Usage.Memory().Value() / (1024 * 1024)) + } + + storages := GetPodStorage(c, pod) + for _, s := range storages { + stroage += uint64(s.Status.Capacity.Storage().Value() / (1024 * 1024 * 1024)) + } + } + appMetrics.CPU = cpu + appMetrics.Memory = memory + appMetrics.Storage = stroage +} + +// LoadApplicationMetrics load application resource metrics +func LoadApplicationMetrics(c client.Client, conf *rest.Config, app *appv1beta1.Application) (*ApplicationMetrics, error) { + appResList, err := ListApplicationResource(c, app.Name, app.Namespace) + if err != nil { + return nil, err + } + components := make([]string, 0) + for _, resource := range appResList { + components = append(components, resource.Object.GetName()) + } + pods := ListApplicationPods(c, app, components) + + appMetrics := &ApplicationMetricsStatus{} + appMetrics.LoadApplicationMetrics(c, conf, pods) + + clusters := make(map[string]struct{}) + nodes := make(map[string]struct{}) + containerNum := 0 + for _, r := range appResList { + clusters[r.Cluster] = struct{}{} + } + for _, pod := range pods { + nodes[pod.Spec.NodeName] = struct{}{} + containerNum += len(pod.Spec.Containers) + } + + appResource := &ApplicationResourceStatus{} + appResource.NodeNum = len(nodes) + appResource.ClusterNum = len(clusters) + appResource.SubresourceNum = len(appResList) + appResource.PodNum = len(pods) + appResource.ContainerNum = containerNum + + return &ApplicationMetrics{ + Status: appMetrics, + Resource: appResource, + }, nil +}