kubevela/references/common/metrics.go

/*
Copyright 2022 The KubeVela Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"context"
	"math"
	"strconv"

	pkgmulticluster "github.com/kubevela/pkg/multicluster"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/rest"
	"k8s.io/metrics/pkg/apis/metrics/v1beta1"
	metricsclientset "k8s.io/metrics/pkg/client/clientset/versioned"
	"sigs.k8s.io/controller-runtime/pkg/client"

	appv1beta1 "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
	"github.com/oam-dev/kubevela/pkg/multicluster"
	"github.com/oam-dev/kubevela/pkg/workflow/providers/legacy/query"
)

const (
	// MetricsNA is the value of metrics when it is not available
	MetricsNA = "N/A"
)

// PodMetricStatus is the status of pod metrics
type PodMetricStatus struct {
	Spec  MetricsLR
	Usage MetricsUsage
}

// ApplicationMetrics is the metrics of application
type ApplicationMetrics struct {
	Metrics     *ApplicationMetricsStatus
	ResourceNum *ApplicationResourceNum
}

// ApplicationMetricsStatus is the status of application metrics
type ApplicationMetricsStatus struct {
	CPUUsage      int64
	CPURequest    int64
	CPULimit      int64
	MemoryUsage   int64
	MemoryRequest int64
	MemoryLimit   int64
	Storage       int64
}

// ApplicationResourceNum is the resource number of application
type ApplicationResourceNum struct {
	Node        int
	Cluster     int
	Subresource int
	Pod         int
	Container   int
}

// MetricsLR is the metric of resource requests and limits
type MetricsLR struct {
	Rcpu, Rmem int64
	Lcpu, Lmem int64
}

// MetricsUsage is the metric of resource usage
type MetricsUsage struct {
	CPU, Mem, Storage int64
}

func podUsage(metrics *v1beta1.PodMetrics) (*resource.Quantity, *resource.Quantity) {
	cpu, mem := new(resource.Quantity), new(resource.Quantity)
	for _, co := range metrics.Containers {
		usage := co.Usage

		if len(usage) == 0 {
			continue
		}
		if usage.Cpu() != nil {
			cpu.Add(*usage.Cpu())
		}
		if co.Usage.Memory() != nil {
			mem.Add(*usage.Memory())
		}
	}
	return cpu, mem
}

func podLimits(spec v1.PodSpec) (*resource.Quantity, *resource.Quantity) {
	cpu, mem := new(resource.Quantity), new(resource.Quantity)
	for _, co := range spec.Containers {
		limits := co.Resources.Limits
		if len(limits) == 0 {
			continue
		}
		if limits.Cpu() != nil {
			cpu.Add(*limits.Cpu())
		}
		if limits.Memory() != nil {
			mem.Add(*limits.Memory())
		}
	}
	return cpu, mem
}

func podRequests(spec v1.PodSpec) (*resource.Quantity, *resource.Quantity) {
	cpu, mem := new(resource.Quantity), new(resource.Quantity)
	for _, co := range spec.Containers {
		req := co.Resources.Requests
		if len(req) == 0 {
			continue
		}
		if req.Cpu() != nil {
			cpu.Add(*req.Cpu())
		}
		if req.Memory() != nil {
			mem.Add(*req.Memory())
		}
	}
	return cpu, mem
}

// ToPercentage computes percentage as string otherwise n/aa.
func ToPercentage(v1, v2 int64) int {
	if v2 == 0 {
		return 0
	}
	return int(math.Floor((float64(v1) / float64(v2)) * 100))
}

// ToPercentageStr computes percentage, but if v2 is 0, it will return NAValue instead of 0.
func ToPercentageStr(v1, v2 int64) string {
	if v2 == 0 {
		return MetricsNA
	}
	return strconv.Itoa(ToPercentage(v1, v2)) + "%"
}

// GetPodMetrics get pod metrics object
func GetPodMetrics(conf *rest.Config, podName, namespace, cluster string) (*v1beta1.PodMetrics, error) {
	ctx := multicluster.ContextWithClusterName(context.Background(), cluster)
	conf.Wrap(pkgmulticluster.NewTransportWrapper())
	metricsClient := metricsclientset.NewForConfigOrDie(conf)
	m, err := metricsClient.MetricsV1beta1().PodMetricses(namespace).Get(ctx, podName, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return m, nil
}

// GetPodResourceSpecAndUsage return the usage metrics of a pod and specified metric including requests and limits metrics
func GetPodResourceSpecAndUsage(c client.Client, pod *v1.Pod, metrics *v1beta1.PodMetrics) (MetricsLR, MetricsUsage) {
	var metricsSpec MetricsLR
	var metricsUsage MetricsUsage
	rcpu, rmem := podRequests(pod.Spec)
	lcpu, lmem := podLimits(pod.Spec)
	metricsSpec.Rcpu, metricsSpec.Lcpu, metricsSpec.Rmem, metricsSpec.Lmem = rcpu.MilliValue(), lcpu.MilliValue(), rmem.Value(), lmem.Value()

	if metrics != nil {
		ccpu, cmem := podUsage(metrics)
		metricsUsage.CPU, metricsUsage.Mem = ccpu.MilliValue(), cmem.Value()
	}

	storage := int64(0)
	storages := GetPodStorage(c, pod)
	for _, s := range storages {
		storage += s.Status.Capacity.Storage().Value()
	}
	metricsUsage.Storage = storage
	return metricsSpec, metricsUsage
}

// GetPodStorage get pod storage
func GetPodStorage(client client.Client, pod *v1.Pod) (storages []v1.PersistentVolumeClaim) {
	for _, v := range pod.Spec.Volumes {
		if v.PersistentVolumeClaim != nil {
			storage := v1.PersistentVolumeClaim{}
			err := client.Get(context.Background(), types.NamespacedName{Name: v.PersistentVolumeClaim.ClaimName, Namespace: pod.Namespace}, &storage)
			if err != nil {
				continue
			}
			storages = append(storages, storage)
		}
	}
	return
}

// ListApplicationResource list application resource
func ListApplicationResource(c client.Client, name, namespace string) ([]query.Resource, error) {
	opt := query.Option{
		Name:      name,
		Namespace: namespace,
		Filter:    query.FilterOption{},
	}
	collector := query.NewAppCollector(c, opt)
	appResList, err := collector.CollectResourceFromApp(context.Background())
	if err != nil {
		return []query.Resource{}, err
	}
	return appResList, err
}

// GetPodOfManagedResource get pod of managed resource
func GetPodOfManagedResource(c client.Client, app *appv1beta1.Application, components string) []*v1.Pod {
	pods := make([]*v1.Pod, 0)
	opt := query.Option{
		Name:      app.Name,
		Namespace: app.Namespace,
		Filter: query.FilterOption{
			Components: []string{components},
			APIVersion: "v1",
			Kind:       "Pod",
		},
		WithTree: true,
	}
	objects, err := CollectApplicationResource(context.Background(), c, opt)
	if err != nil {
		return pods
	}
	for _, object := range objects {
		pod := &v1.Pod{}
		err = runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), pod)
		if err != nil {
			continue
		}
		pods = append(pods, pod)
	}
	return pods
}

// GetPodMetricsStatus get pod metrics
func GetPodMetricsStatus(c client.Client, conf *rest.Config, pod *v1.Pod, cluster string) (*PodMetricStatus, error) {
	metricsStatus := &PodMetricStatus{}
	podMetrics, err := GetPodMetrics(conf, pod.Name, pod.Namespace, cluster)
	if err != nil {
		return nil, err
	}
	spec, usage := GetPodResourceSpecAndUsage(c, pod, podMetrics)
	metricsStatus.Spec = spec
	metricsStatus.Usage = usage

	return metricsStatus, nil
}

// GetApplicationMetrics get application metrics
func GetApplicationMetrics(c client.Client, conf *rest.Config, app *appv1beta1.Application) (*ApplicationMetrics, error) {
	appResList, err := ListApplicationResource(c, app.Name, app.Namespace)
	if err != nil {
		return nil, err
	}
	clusters := make(map[string]struct{})
	nodes := make(map[string]struct{})
	podNum := 0
	containerNum := 0
	podMetricsArray := make([]*PodMetricStatus, 0)

	for _, managedResource := range appResList {
		clusters[managedResource.Cluster] = struct{}{}
		pods := GetPodOfManagedResource(c, app, managedResource.Object.GetName())
		podNum += len(pods)
		for _, pod := range pods {
			nodes[pod.Spec.NodeName] = struct{}{}
			containerNum += len(pod.Spec.Containers)
			status, err := GetPodMetricsStatus(c, conf, pod, managedResource.Cluster)
			if err != nil {
				continue
			}
			podMetricsArray = append(podMetricsArray, status)
		}
	}

	appResource := &ApplicationResourceNum{
		Cluster:     len(clusters),
		Node:        len(nodes),
		Subresource: len(appResList),
		Pod:         podNum,
		Container:   containerNum,
	}
	appMetrics := &ApplicationMetricsStatus{}
	for _, metrics := range podMetricsArray {
		appMetrics.CPUUsage += metrics.Usage.CPU
		appMetrics.CPULimit += metrics.Spec.Lcpu
		appMetrics.CPURequest += metrics.Spec.Rcpu
		appMetrics.MemoryUsage += metrics.Usage.Mem
		appMetrics.MemoryLimit += metrics.Spec.Lmem
		appMetrics.MemoryRequest += metrics.Spec.Rmem
		appMetrics.Storage += metrics.Usage.Storage
	}
	return &ApplicationMetrics{
		Metrics:     appMetrics,
		ResourceNum: appResource,
	}, nil
}