mirror of
https://github.com/replicatedhq/troubleshoot.git
synced 2026-04-15 07:16:34 +00:00
* Add collect command and remote host collectors
Adds the ability to run a host collector on a set of remote k8s nodes.
Target nodes can be filtered using the --selector flag, with the same
syntax as kubectl. Existing flags for --collector-image,
--collector-pullpolicy and --request-timeout are used. To run on a
specified node, --selector="kubernetes.io/hostname=kind-worker2" could
be used.
The collect command is used by the remote collector to output the
results using a "raw" format, which uses the filename as the key, and
the value the output as a escaped json string. When run manually it
defaults to fully decoded json. The existing block devices,
ipv4interfaces and services host collectors don't decode properly - the
fix is to convert their slice output to a map (fix not included as
unsure what depends on the existing format).
The collect command is also useful for troubleshooting preflight issues.
Examples are included to show remote collector usage.
```
bin/collect --collector-image=croomes/troubleshoot:latest examples/collect/remote/memory.yaml --namespace test
{
"kind-control-plane": {
"system/memory.json": {
"total": 1304207360
}
},
"kind-worker": {
"system/memory.json": {
"total": 1695780864
}
},
"kind-worker2": {
"system/memory.json": {
"total": 1726353408
}
}
}
```
The preflight command has been updated to run remote collectors. To run
a host collector remotely it must be specified in the spec as a
`remoteCollector`:
```
apiVersion: troubleshoot.sh/v1beta2
kind: HostPreflight
metadata:
name: memory
spec:
remoteCollectors:
- memory:
collectorName: memory
analyzers:
- memory:
outcomes:
- fail:
when: "< 8Gi"
message: At least 8Gi of memory is required
- warn:
when: "< 32Gi"
message: At least 32Gi of memory is recommended
- pass:
message: The system has as sufficient memory
```
Results for each node are analyzed separately, with the node name
appended to the title:
```
bin/preflight --interactive=false --collector-image=croomes/troubleshoot:latest examples/preflight/remote/memory.yaml --format=json
{memory running 0 1}
{memory completed 1 1}
{
"fail": [
{
"title": "Amount of Memory (kind-worker2)",
"message": "At least 8Gi of memory is required"
},
{
"title": "Amount of Memory (kind-worker)",
"message": "At least 8Gi of memory is required"
},
{
"title": "Amount of Memory (kind-control-plane)",
"message": "At least 8Gi of memory is required"
}
]
}
```
Also added a host collector to allow preflight checks of required kernel
modules, which is the main driver for this change.
381 lines
8.7 KiB
Go
381 lines
8.7 KiB
Go
package analyzer
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
|
|
"github.com/pkg/errors"
|
|
troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
|
|
"github.com/replicatedhq/troubleshoot/pkg/multitype"
|
|
)
|
|
|
|
type AnalyzeResult struct {
|
|
IsPass bool
|
|
IsFail bool
|
|
IsWarn bool
|
|
|
|
Title string
|
|
Message string
|
|
URI string
|
|
IconKey string
|
|
IconURI string
|
|
}
|
|
|
|
type getCollectedFileContents func(string) ([]byte, error)
|
|
type getChildCollectedFileContents func(string) (map[string][]byte, error)
|
|
|
|
func isExcluded(excludeVal multitype.BoolOrString) (bool, error) {
|
|
if excludeVal.Type == multitype.Bool {
|
|
return excludeVal.BoolVal, nil
|
|
}
|
|
|
|
if excludeVal.StrVal == "" {
|
|
return false, nil
|
|
}
|
|
|
|
parsed, err := strconv.ParseBool(excludeVal.StrVal)
|
|
if err != nil {
|
|
return false, errors.Wrap(err, "failed to parse bool string")
|
|
}
|
|
|
|
return parsed, nil
|
|
}
|
|
|
|
func HostAnalyze(hostAnalyzer *troubleshootv1beta2.HostAnalyze, getFile getCollectedFileContents, findFiles getChildCollectedFileContents) []*AnalyzeResult {
|
|
analyzer, ok := GetHostAnalyzer(hostAnalyzer)
|
|
if !ok {
|
|
return NewAnalyzeResultError(analyzer, errors.New("invalid host analyzer"))
|
|
}
|
|
|
|
isExcluded, _ := analyzer.IsExcluded()
|
|
if isExcluded {
|
|
return nil
|
|
}
|
|
|
|
result, err := analyzer.Analyze(getFile)
|
|
if err != nil {
|
|
return NewAnalyzeResultError(analyzer, errors.Wrap(err, "analyze"))
|
|
}
|
|
return result
|
|
}
|
|
|
|
func NewAnalyzeResultError(analyzer HostAnalyzer, err error) []*AnalyzeResult {
|
|
if analyzer != nil {
|
|
return []*AnalyzeResult{{
|
|
IsFail: true,
|
|
Title: analyzer.Title(),
|
|
Message: fmt.Sprintf("Analyzer Failed: %v", err),
|
|
}}
|
|
}
|
|
return []*AnalyzeResult{{
|
|
IsFail: true,
|
|
Title: "nil analyzer",
|
|
Message: fmt.Sprintf("Analyzer Failed: %v", err),
|
|
}}
|
|
}
|
|
|
|
func Analyze(analyzer *troubleshootv1beta2.Analyze, getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
|
|
if analyzer == nil {
|
|
return nil, errors.New("nil analyzer")
|
|
}
|
|
|
|
if analyzer.ClusterVersion != nil {
|
|
isExcluded, err := isExcluded(analyzer.ClusterVersion.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeClusterVersion(analyzer.ClusterVersion, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.StorageClass != nil {
|
|
isExcluded, err := isExcluded(analyzer.StorageClass.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeStorageClass(analyzer.StorageClass, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.CustomResourceDefinition != nil {
|
|
isExcluded, err := isExcluded(analyzer.CustomResourceDefinition.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeCustomResourceDefinition(analyzer.CustomResourceDefinition, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.Ingress != nil {
|
|
isExcluded, err := isExcluded(analyzer.Ingress.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeIngress(analyzer.Ingress, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.Secret != nil {
|
|
isExcluded, err := isExcluded(analyzer.Secret.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeSecret(analyzer.Secret, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.ConfigMap != nil {
|
|
isExcluded, err := isExcluded(analyzer.ConfigMap.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeConfigMap(analyzer.ConfigMap, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.ImagePullSecret != nil {
|
|
isExcluded, err := isExcluded(analyzer.ImagePullSecret.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeImagePullSecret(analyzer.ImagePullSecret, findFiles)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.DeploymentStatus != nil {
|
|
isExcluded, err := isExcluded(analyzer.DeploymentStatus.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeDeploymentStatus(analyzer.DeploymentStatus, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.StatefulsetStatus != nil {
|
|
isExcluded, err := isExcluded(analyzer.StatefulsetStatus.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeStatefulsetStatus(analyzer.StatefulsetStatus, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.ContainerRuntime != nil {
|
|
isExcluded, err := isExcluded(analyzer.ContainerRuntime.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeContainerRuntime(analyzer.ContainerRuntime, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.Distribution != nil {
|
|
isExcluded, err := isExcluded(analyzer.Distribution.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeDistribution(analyzer.Distribution, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.NodeResources != nil {
|
|
isExcluded, err := isExcluded(analyzer.NodeResources.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeNodeResources(analyzer.NodeResources, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.TextAnalyze != nil {
|
|
isExcluded, err := isExcluded(analyzer.TextAnalyze.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
multiResult, err := analyzeTextAnalyze(analyzer.TextAnalyze, findFiles)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return multiResult, nil
|
|
}
|
|
if analyzer.Postgres != nil {
|
|
isExcluded, err := isExcluded(analyzer.Postgres.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzePostgres(analyzer.Postgres, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.Mysql != nil {
|
|
isExcluded, err := isExcluded(analyzer.Mysql.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeMysql(analyzer.Mysql, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.Redis != nil {
|
|
isExcluded, err := isExcluded(analyzer.Redis.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeRedis(analyzer.Redis, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.CephStatus != nil {
|
|
isExcluded, err := isExcluded(analyzer.CephStatus.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := cephStatus(analyzer.CephStatus, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if analyzer.Longhorn != nil {
|
|
isExcluded, err := isExcluded(analyzer.Longhorn.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
return longhorn(analyzer.Longhorn, getFile, findFiles)
|
|
}
|
|
|
|
if analyzer.RegistryImages != nil {
|
|
isExcluded, err := isExcluded(analyzer.RegistryImages.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeRegistry(analyzer.RegistryImages, getFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
|
|
if analyzer.WeaveReport != nil {
|
|
isExcluded, err := isExcluded(analyzer.WeaveReport.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
results, err := analyzeWeaveReport(analyzer.WeaveReport, findFiles)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
if analyzer.Sysctl != nil {
|
|
isExcluded, err := isExcluded(analyzer.Sysctl.Exclude)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isExcluded {
|
|
return nil, nil
|
|
}
|
|
result, err := analyzeSysctl(analyzer.Sysctl, findFiles)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if result == nil {
|
|
return []*AnalyzeResult{}, nil
|
|
}
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
|
|
return nil, errors.New("invalid analyzer")
|
|
}
|