github/troubleshoot
1
0
Fork 0
mirror of https://github.com/replicatedhq/troubleshoot.git synced 2026-02-14 18:29:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add k8s custom metrics collector (#1174)

Browse Source 
Collector that collects custom k8s metrics from custom.metrics.k8s.io/v1beta1/ and saves them in the bundle under the /metrics directory
This commit is contained in:
Ahmed Mousa
2023-06-09 11:26:03 +02:00
committed by GitHub
parent 60d5b686cc
commit 620fa75eb5
13 changed files with 483 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
config/crds/troubleshoot.sh_collectors.yaml
View File

@@ -245,6 +245,36 @@ spec:
- image
- namespace
type: object
customMetrics:
properties:
collectorName:
type: string
exclude:
type: BoolString
metricRequests:
items:
description: MetricRequest the details of the MetricValuesList
to be retrieved
properties:
namespace:
description: Namespace for which to collect the metric
values, empty for non-namespaces resources.
type: string
objectName:
description: ObjectName for which to collect metric
values, all resources when empty. Note that for
namespaced resources a Namespace has to be supplied
regardless.
type: string
resourceMetricName:
description: ResourceMetricName name of the MetricValueList
as per the APIResourceList from custom.metrics.k8s.io/v1beta1
type: string
required:
- resourceMetricName
type: object
type: array
type: object
data:
properties:
collectorName:

30
config/crds/troubleshoot.sh_preflights.yaml
View File

@@ -1740,6 +1740,36 @@ spec:
- image
- namespace
type: object
customMetrics:
properties:
collectorName:
type: string
exclude:
type: BoolString
metricRequests:
items:
description: MetricRequest the details of the MetricValuesList
to be retrieved
properties:
namespace:
description: Namespace for which to collect the metric
values, empty for non-namespaces resources.
type: string
objectName:
description: ObjectName for which to collect metric
values, all resources when empty. Note that for
namespaced resources a Namespace has to be supplied
regardless.
type: string
resourceMetricName:
description: ResourceMetricName name of the MetricValueList
as per the APIResourceList from custom.metrics.k8s.io/v1beta1
type: string
required:
- resourceMetricName
type: object
type: array
type: object
data:
properties:
collectorName:

30
config/crds/troubleshoot.sh_supportbundles.yaml
View File

@@ -1771,6 +1771,36 @@ spec:
- image
- namespace
type: object
customMetrics:
properties:
collectorName:
type: string
exclude:
type: BoolString
metricRequests:
items:
description: MetricRequest the details of the MetricValuesList
to be retrieved
properties:
namespace:
description: Namespace for which to collect the metric
values, empty for non-namespaces resources.
type: string
objectName:
description: ObjectName for which to collect metric
values, all resources when empty. Note that for
namespaced resources a Namespace has to be supplied
regardless.
type: string
resourceMetricName:
description: ResourceMetricName name of the MetricValueList
as per the APIResourceList from custom.metrics.k8s.io/v1beta1
type: string
required:
- resourceMetricName
type: object
type: array
type: object
data:
properties:
collectorName:

1
go.mod
View File

@@ -211,6 +211,7 @@ require (
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
k8s.io/metrics v0.27.2
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
periph.io/x/host/v3 v3.8.2
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

2
go.sum
View File

@@ -1476,6 +1476,8 @@ k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg=
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg=
k8s.io/metrics v0.27.2 h1:TD6z3dhhN9bgg5YkbTh72bPiC1BsxipBLPBWyC3VQAU=
k8s.io/metrics v0.27.2/go.mod h1:v3OT7U0DBvoAzWVzGZWQhdV4qsRJWchzs/LeVN8bhW4=
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
oras.land/oras-go v1.2.3 h1:v8PJl+gEAntI1pJ/LCrDgsuk+1PKVavVEPsYIHFE5uY=

18
pkg/apis/troubleshoot/v1beta2/collector_shared.go
View File

@@ -27,6 +27,23 @@ type ClusterResources struct {
IgnoreRBAC bool `json:"ignoreRBAC,omitempty" yaml:"ignoreRBAC"`
}
// MetricRequest the details of the MetricValuesList to be retrieved
type MetricRequest struct {
// Namespace for which to collect the metric values, empty for non-namespaces resources.
Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"`
// ObjectName for which to collect metric values, all resources when empty.
// Note that for namespaced resources a Namespace has to be supplied regardless.
ObjectName string `json:"objectName,omitempty" yaml:"objectName,omitempty"`
// ResourceMetricName name of the MetricValueList as per the APIResourceList from
// custom.metrics.k8s.io/v1beta1
ResourceMetricName string `json:"resourceMetricName" yaml:"resourceMetricName"`
}
type CustomMetrics struct {
CollectorMeta `json:",inline" yaml:",inline"`
MetricRequests []MetricRequest `json:"metricRequests,omitempty" yaml:"metricRequests,omitempty"`
}
type Secret struct {
CollectorMeta `json:",inline" yaml:",inline"`
Name string `json:"name,omitempty" yaml:"name,omitempty"`
@@ -231,6 +248,7 @@ type Collect struct {
ClusterInfo *ClusterInfo `json:"clusterInfo,omitempty" yaml:"clusterInfo,omitempty"`
ClusterResources *ClusterResources `json:"clusterResources,omitempty" yaml:"clusterResources,omitempty"`
Secret *Secret `json:"secret,omitempty" yaml:"secret,omitempty"`
CustomMetrics *CustomMetrics `json:"customMetrics,omitempty" yaml:"customMetrics,omitempty"`
ConfigMap *ConfigMap `json:"configMap,omitempty" yaml:"configMap,omitempty"`
Logs *Logs `json:"logs,omitempty" yaml:"logs,omitempty"`
Run *Run `json:"run,omitempty" yaml:"run,omitempty"`

41
pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go
View File

@@ -782,6 +782,11 @@ func (in *Collect) DeepCopyInto(out *Collect) {
*out = new(Secret)
(*in).DeepCopyInto(*out)
}
if in.CustomMetrics != nil {
in, out := &in.CustomMetrics, &out.CustomMetrics
*out = new(CustomMetrics)
(*in).DeepCopyInto(*out)
}
if in.ConfigMap != nil {
in, out := &in.ConfigMap, &out.ConfigMap
*out = new(ConfigMap)
@@ -1158,6 +1163,27 @@ func (in *CopyFromHost) DeepCopy() *CopyFromHost {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CustomMetrics) DeepCopyInto(out *CustomMetrics) {
*out = *in
in.CollectorMeta.DeepCopyInto(&out.CollectorMeta)
if in.MetricRequests != nil {
in, out := &in.MetricRequests, &out.MetricRequests
*out = make([]MetricRequest, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomMetrics.
func (in *CustomMetrics) DeepCopy() *CustomMetrics {
if in == nil {
return nil
}
out := new(CustomMetrics)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CustomResourceDefinition) DeepCopyInto(out *CustomResourceDefinition) {
*out = *in
@@ -2738,6 +2764,21 @@ func (in *MemoryAnalyze) DeepCopy() *MemoryAnalyze {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *MetricRequest) DeepCopyInto(out *MetricRequest) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricRequest.
func (in *MetricRequest) DeepCopy() *MetricRequest {
if in == nil {
return nil
}
out := new(MetricRequest)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeResourceFilters) DeepCopyInto(out *NodeResourceFilters) {
*out = *in

5
pkg/collect/collector.go
View File

@@ -63,6 +63,8 @@ func GetCollector(collector *troubleshootv1beta2.Collect, bundlePath string, nam
return &CollectClusterInfo{collector.ClusterInfo, bundlePath, namespace, clientConfig, RBACErrors}, true
case collector.ClusterResources != nil:
return &CollectClusterResources{collector.ClusterResources, bundlePath, namespace, clientConfig, RBACErrors}, true
case collector.CustomMetrics != nil:
return &CollectMetrics{collector.CustomMetrics, bundlePath, clientConfig, client, ctx, RBACErrors}, true
case collector.Secret != nil:
return &CollectSecret{collector.Secret, bundlePath, namespace, clientConfig, client, ctx, RBACErrors}, true
case collector.ConfigMap != nil:
@@ -116,6 +118,9 @@ func getCollectorName(c interface{}) string {
collector = "cluster-info"
case *CollectClusterResources:
collector = "cluster-resources"
case *CollectMetrics:
collector = "custom-metrics"
name = v.Collector.CollectorName
case *CollectSecret:
collector = "secret"
name = v.Collector.CollectorName

130
pkg/collect/k8s_metrics.go Normal file
View File

@@ -0,0 +1,130 @@
package collect
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/url"
"path/filepath"
"strings"
"github.com/pkg/errors"
troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
"k8s.io/metrics/pkg/apis/custom_metrics"
)
const (
namespaceSingular = "namespace"
namespacePlural = "namespaces"
urlBase = "/apis/custom.metrics.k8s.io/v1beta1"
metricsErrorFile = "metrics/errors.json"
)
type CollectMetrics struct {
Collector *troubleshootv1beta2.CustomMetrics
BundlePath string
ClientConfig *rest.Config
Client kubernetes.Interface
Context context.Context
RBACErrors
}
func (c *CollectMetrics) Title() string {
return getCollectorName(c)
}
func (c *CollectMetrics) IsExcluded() (bool, error) {
return isExcluded(c.Collector.Exclude)
}
func (c *CollectMetrics) Collect(progressChan chan<- interface{}) (CollectorResult, error) {
output := NewResult()
resultLists := make(map[string][]custom_metrics.MetricValue)
errorsList := make([]string, 0)
for _, metricRequest := range c.Collector.MetricRequests {
klog.V(2).Infof("Getting metric values: %+v\n", metricRequest.ResourceMetricName)
endpoint, metricName, err := constructEndpoint(metricRequest)
if err != nil {
errorsList = append(errorsList, errors.Wrapf(err, "could not construct endpoint for %s", metricRequest.ResourceMetricName).Error())
continue
}
klog.V(2).Infof("Querying: %+v\n", endpoint)
response, err := c.Client.CoreV1().RESTClient().Get().AbsPath(endpoint).DoRaw(c.Context)
if err != nil {
errorsList = append(errorsList, errors.Wrapf(err, "could not query endpoint %s", endpoint).Error())
continue
}
metricsValues := custom_metrics.MetricValueList{}
json.Unmarshal(response, &metricsValues)
// metrics
// |_ <resource_type>
// |_ <metric_name>
// |_ <namespace>.json or <non_namespaced_object>.json
var path []string
for _, item := range metricsValues.Items {
if item.DescribedObject.Namespace != "" {
path = []string{"metrics", item.DescribedObject.Kind, metricName, fmt.Sprintf("%s.json", item.DescribedObject.Namespace)}
} else {
path = []string{"metrics", item.DescribedObject.Kind, metricName, fmt.Sprintf("%s.json", item.DescribedObject.Name)}
}
filePath := filepath.Join(path...)
if _, ok := resultLists[filePath]; !ok {
resultLists[filePath] = make([]custom_metrics.MetricValue, 0)
}
resultLists[filePath] = append(resultLists[filePath], item)
}
}
// Construct output.
for relativePath, list := range resultLists {
payload, err := json.MarshalIndent(list, "", " ")
if err != nil {
klog.V(2).Infof("Could not parse for: %+v\n", relativePath)
errorsList = append(errorsList, errors.Wrapf(err, "could not format readings for %s", relativePath).Error())
}
output.SaveResult(c.BundlePath, relativePath, bytes.NewBuffer(payload))
}
errPayload := marshalErrors(errorsList)
output.SaveResult(c.BundlePath, metricsErrorFile, errPayload)
return output, nil
}
func constructEndpoint(metricRequest troubleshootv1beta2.MetricRequest) (string, string, error) {
metricNameComponents := strings.Split(metricRequest.ResourceMetricName, "/")
if len(metricNameComponents) != 2 {
return "", "", errors.New("wrong metric name format %s")
}
objectType := metricNameComponents[0]
// Namespace related metrics are grouped under singular format "namespace/"
// unlike other resources.
if objectType == namespacePlural {
objectType = namespaceSingular
}
metricName := metricNameComponents[1]
objectSelector := "*"
if metricRequest.ObjectName != "" {
objectSelector = metricRequest.ObjectName
}
var endpoint string
var err error
if metricRequest.Namespace != "" {
// namespaced objects
// endpoint <resource_type>/namespaces/<namespace>/<resrouce_name or *>/<metric>
endpoint, err = url.JoinPath(urlBase, namespacePlural, metricRequest.Namespace, objectType, objectSelector, metricName)
if err != nil {
return "", "", errors.Wrap(err, "could not construct url")
}
} else {
// non-namespaced objects
// endpoint <resource_type>/<resrouce_name or *>/<metric>
endpoint, err = url.JoinPath(urlBase, objectType, objectSelector, metricName)
if err != nil {
return "", "", errors.Wrap(err, "could not construct url")
}
}
return endpoint, metricName, nil
}

91
pkg/collect/k8s_metrics_test.go Normal file
View File

@@ -0,0 +1,91 @@
package collect
import (
"testing"
"github.com/pkg/errors"
troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConstructEndpoint(t *testing.T) {
// Define test cases
testCases := []struct {
name string
metricRequest troubleshootv1beta2.MetricRequest
expectedEndpoint string
expectedMetric string
expectedError error
}{
{
name: "Namespaced object with namespace and object name",
metricRequest: troubleshootv1beta2.MetricRequest{
Namespace: "namespace",
ObjectName: "object",
ResourceMetricName: "pods/metric",
},
expectedEndpoint: "/apis/custom.metrics.k8s.io/v1beta1/namespaces/namespace/pods/object/metric",
expectedMetric: "metric",
expectedError: nil,
},
{
name: "Namespaced object with namespace and empty object name",
metricRequest: troubleshootv1beta2.MetricRequest{
Namespace: "namespace",
ObjectName: "",
ResourceMetricName: "pods/metric",
},
expectedEndpoint: "/apis/custom.metrics.k8s.io/v1beta1/namespaces/namespace/pods/*/metric",
expectedMetric: "metric",
expectedError: nil,
},
{
name: "Non-namespaced object",
metricRequest: troubleshootv1beta2.MetricRequest{
ResourceMetricName: "nodes/metric",
ObjectName: "object",
},
expectedEndpoint: "/apis/custom.metrics.k8s.io/v1beta1/nodes/object/metric",
expectedMetric: "metric",
expectedError: nil,
},
{
name: "Non-namespaced object with empty object name",
metricRequest: troubleshootv1beta2.MetricRequest{
ResourceMetricName: "namespaces/metric",
ObjectName: "",
},
expectedEndpoint: "/apis/custom.metrics.k8s.io/v1beta1/namespace/*/metric",
expectedMetric: "metric",
expectedError: nil,
},
{
name: "Invalid metric name format",
metricRequest: troubleshootv1beta2.MetricRequest{
ResourceMetricName: "invalid-metric-name",
ObjectName: "object",
},
expectedEndpoint: "",
expectedMetric: "",
expectedError: errors.New("wrong metric name format"),
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// Call the function under test
endpoint, metric, err := constructEndpoint(tc.metricRequest)
// Verify the results
if tc.expectedError != nil {
require.Error(t, err)
assert.Contains(t, err.Error(), tc.expectedError.Error())
} else {
require.NoError(t, err)
assert.Equal(t, tc.expectedEndpoint, endpoint)
assert.Equal(t, tc.expectedMetric, metric)
}
})
}
}

35
schemas/collector-troubleshoot-v1beta2.json
View File

@@ -335,6 +335,41 @@
}
}
},
"customMetrics": {
"type": "object",
"properties": {
"collectorName": {
"type": "string"
},
"exclude": {
"oneOf": [{"type": "string"},{"type": "boolean"}]
},
"metricRequests": {
"type": "array",
"items": {
"description": "MetricRequest the details of the MetricValuesList to be retrieved",
"type": "object",
"required": [
"resourceMetricName"
],
"properties": {
"namespace": {
"description": "Namespace for which to collect the metric values, empty for non-namespaces resources.",
"type": "string"
},
"objectName": {
"description": "ObjectName for which to collect metric values, all resources when empty. Note that for namespaced resources a Namespace has to be supplied regardless.",
"type": "string"
},
"resourceMetricName": {
"description": "ResourceMetricName name of the MetricValueList as per the APIResourceList from custom.metrics.k8s.io/v1beta1",
"type": "string"
}
}
}
}
}
},
"data": {
"type": "object",
"required": [

35
schemas/preflight-troubleshoot-v1beta2.json
View File

@@ -2638,6 +2638,41 @@
}
}
},
"customMetrics": {
"type": "object",
"properties": {
"collectorName": {
"type": "string"
},
"exclude": {
"oneOf": [{"type": "string"},{"type": "boolean"}]
},
"metricRequests": {
"type": "array",
"items": {
"description": "MetricRequest the details of the MetricValuesList to be retrieved",
"type": "object",
"required": [
"resourceMetricName"
],
"properties": {
"namespace": {
"description": "Namespace for which to collect the metric values, empty for non-namespaces resources.",
"type": "string"
},
"objectName": {
"description": "ObjectName for which to collect metric values, all resources when empty. Note that for namespaced resources a Namespace has to be supplied regardless.",
"type": "string"
},
"resourceMetricName": {
"description": "ResourceMetricName name of the MetricValueList as per the APIResourceList from custom.metrics.k8s.io/v1beta1",
"type": "string"
}
}
}
}
}
},
"data": {
"type": "object",
"required": [

35
schemas/supportbundle-troubleshoot-v1beta2.json
View File

@@ -2684,6 +2684,41 @@
}
}
},
"customMetrics": {
"type": "object",
"properties": {
"collectorName": {
"type": "string"
},
"exclude": {
"oneOf": [{"type": "string"},{"type": "boolean"}]
},
"metricRequests": {
"type": "array",
"items": {
"description": "MetricRequest the details of the MetricValuesList to be retrieved",
"type": "object",
"required": [
"resourceMetricName"
],
"properties": {
"namespace": {
"description": "Namespace for which to collect the metric values, empty for non-namespaces resources.",
"type": "string"
},
"objectName": {
"description": "ObjectName for which to collect metric values, all resources when empty. Note that for namespaced resources a Namespace has to be supplied regardless.",
"type": "string"
},
"resourceMetricName": {
"description": "ResourceMetricName name of the MetricValueList as per the APIResourceList from custom.metrics.k8s.io/v1beta1",
"type": "string"
}
}
}
}
}
},
"data": {
"type": "object",
"required": [