From 2bf19eaddf0731c9fb2b07465d6e9d934c73e554 Mon Sep 17 00:00:00 2001 From: emosbaugh Date: Fri, 13 Nov 2020 09:12:42 -0800 Subject: [PATCH] Ceph collectors and analyzers (#295) * Ceph collectors and analyzers * updating based on prior pr * fixes * fixes --- go.mod | 1 + go.sum | 7 + pkg/analyze/analyzer.go | 14 ++ pkg/analyze/ceph.go | 175 +++++++++++++++ pkg/analyze/ceph_test.go | 204 ++++++++++++++++++ .../troubleshoot/v1beta2/analyzer_shared.go | 8 + .../troubleshoot/v1beta2/collector_shared.go | 12 ++ .../v1beta2/zz_generated.deepcopy.go | 53 +++++ pkg/collect/ceph.go | 159 ++++++++++++++ pkg/collect/collector.go | 9 + 10 files changed, 642 insertions(+) create mode 100644 pkg/analyze/ceph.go create mode 100644 pkg/analyze/ceph_test.go create mode 100644 pkg/collect/ceph.go diff --git a/go.mod b/go.mod index 67169944..1ab16666 100644 --- a/go.mod +++ b/go.mod @@ -51,4 +51,5 @@ require ( k8s.io/cli-runtime v0.18.0 k8s.io/client-go v0.18.2 sigs.k8s.io/controller-runtime v0.5.1-0.20200402191424-df180accb901 + sigs.k8s.io/controller-tools v0.3.0 // indirect ) diff --git a/go.sum b/go.sum index 021e0ed1..37ccd489 100644 --- a/go.sum +++ b/go.sum @@ -191,6 +191,8 @@ github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRf github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/flect v0.2.0 h1:EWCvMGGxOjsgwlWaP+f4+Hh6yrrte7JeFL2S6b+0hdM= +github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -325,6 +327,7 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/manifoldco/promptui v0.3.2 h1:rir7oByTERac6jhpHUPErHuopoRDvO3jxS+FdadEns8= github.com/manifoldco/promptui v0.3.2/go.mod h1:8JU+igZ+eeiiRku4T5BjtKh2ms8sziGpSYl1gN8Bazw= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -688,6 +691,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966 h1:B0J02caTR6tpSJozBJyiAzT6CtBzjclw4pgm9gg8Ys0= +gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -732,6 +737,8 @@ k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/controller-runtime v0.5.1-0.20200402191424-df180accb901 h1:qwHvTyQQBjATQeKPdJ0TwcXdtScjyI6GuiAQ5CSqEvM= sigs.k8s.io/controller-runtime v0.5.1-0.20200402191424-df180accb901/go.mod h1:j4echH3Y/UPHRpXS65rxGXujda8iWOheMQvDh1uNgaY= +sigs.k8s.io/controller-tools v0.3.0 h1:y3YD99XOyWaXkiF1kd41uRvfp/64teWcrEZFuHxPhJ4= +sigs.k8s.io/controller-tools v0.3.0/go.mod h1:enhtKGfxZD1GFEoMgP8Fdbu+uKQ/cq1/WGJhdVChfvI= sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/pkg/analyze/analyzer.go b/pkg/analyze/analyzer.go index 8c4a6b10..775d3fea 100644 --- a/pkg/analyze/analyzer.go +++ b/pkg/analyze/analyzer.go @@ -251,6 +251,20 @@ func Analyze(analyzer *troubleshootv1beta2.Analyze, getFile getCollectedFileCont } return []*AnalyzeResult{result}, nil } + if analyzer.CephStatus != nil { + isExcluded, err := isExcluded(analyzer.CephStatus.Exclude) + if err != nil { + return nil, err + } + if isExcluded { + return nil, nil + } + result, err := cephStatus(analyzer.CephStatus, getFile) + if err != nil { + return nil, err + } + return []*AnalyzeResult{result}, nil + } return nil, errors.New("invalid analyzer") } diff --git a/pkg/analyze/ceph.go b/pkg/analyze/ceph.go new file mode 100644 index 00000000..80bc1891 --- /dev/null +++ b/pkg/analyze/ceph.go @@ -0,0 +1,175 @@ +package analyzer + +import ( + "encoding/json" + "path" + "strings" + + "github.com/pkg/errors" + troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" + "github.com/replicatedhq/troubleshoot/pkg/collect" +) + +type CephHealth string + +const ( + CephHealthOK CephHealth = "HEALTH_OK" + CephHealthWarn CephHealth = "HEALTH_WARN" + CephHealthErr CephHealth = "HEALTH_ERR" +) + +func (a CephHealth) Compare(b CephHealth) int { + if a == b { + return 0 + } + switch a { + case CephHealthOK: + return 1 + case CephHealthWarn: + switch b { + case CephHealthOK: + return -1 + case CephHealthErr: + return 1 + } + return 1 + case CephHealthErr: + switch b { + case CephHealthOK, CephHealthWarn: + return -1 + } + return 1 + default: + return -1 + } +} + +var CephStatusDefaultOutcomes = []*troubleshootv1beta2.Outcome{ + { + Pass: &troubleshootv1beta2.SingleOutcome{ + Message: "Ceph is healthy", + }, + }, + { + Warn: &troubleshootv1beta2.SingleOutcome{ + Message: "Ceph status is HEALTH_WARN", + URI: "https://rook.io/docs/rook/v1.4/ceph-common-issues.html", + }, + }, + { + Fail: &troubleshootv1beta2.SingleOutcome{ + Message: "Ceph status is HEALTH_ERR", + URI: "https://rook.io/docs/rook/v1.4/ceph-common-issues.html", + }, + }, +} + +func cephStatus(analyzer *troubleshootv1beta2.CephStatusAnalyze, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) { + fileName := path.Join(collect.GetCephCollectorFilepath(analyzer.CollectorName, analyzer.Namespace), "status.json") + collected, err := getCollectedFileContents(fileName) + if err != nil { + return nil, errors.Wrap(err, "failed to read collected ceph status") + } + + title := analyzer.CheckName + if title == "" { + title = "Ceph Status" + } + + analyzeResult := &AnalyzeResult{ + Title: title, + IconKey: "rook", // maybe this should be ceph? + IconURI: "https://troubleshoot.sh/images/analyzer-icons/rook.svg?w=11&h=16", + } + + status := struct { + Health struct { + Status string `json:"status"` + } `json:"health"` + }{} + if err := json.Unmarshal(collected, &status); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal status.json") + } + + if len(analyzer.Outcomes) == 0 { + analyzer.Outcomes = CephStatusDefaultOutcomes + } + + for _, outcome := range analyzer.Outcomes { + if outcome.Fail != nil { + if outcome.Fail.When == "" { + outcome.Fail.When = string(CephHealthErr) + } + match, err := compareCephStatus(status.Health.Status, outcome.Fail.When) + if err != nil { + return nil, errors.Wrap(err, "failed to compare ceph status") + } else if match { + analyzeResult.IsFail = true + analyzeResult.Message = outcome.Fail.Message + analyzeResult.URI = outcome.Fail.URI + return analyzeResult, nil + } + } else if outcome.Warn != nil { + if outcome.Warn.When == "" { + outcome.Warn.When = string(CephHealthWarn) + } + match, err := compareCephStatus(status.Health.Status, outcome.Warn.When) + if err != nil { + return nil, errors.Wrap(err, "failed to compare ceph status") + } else if match { + analyzeResult.IsWarn = true + analyzeResult.Message = outcome.Warn.Message + analyzeResult.URI = outcome.Warn.URI + return analyzeResult, nil + } + } else if outcome.Pass != nil { + if outcome.Pass.When == "" { + outcome.Pass.When = string(CephHealthOK) + } + match, err := compareCephStatus(status.Health.Status, outcome.Pass.When) + if err != nil { + return nil, errors.Wrap(err, "failed to compare ceph status") + } else if match { + analyzeResult.IsPass = true + analyzeResult.Message = outcome.Pass.Message + analyzeResult.URI = outcome.Pass.URI + return analyzeResult, nil + } + } + } + + return analyzeResult, nil +} + +func compareCephStatus(actual, when string) (bool, error) { + parts := strings.Split(strings.TrimSpace(when), " ") + + if len(parts) == 1 { + value := strings.TrimSpace(parts[0]) + return value == actual, nil + } + + if len(parts) != 2 { + return false, errors.New("unable to parse when range") + } + + operator := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + compareResult := CephHealth(actual).Compare(CephHealth(value)) + + switch operator { + case "=", "==", "===": + return compareResult == 0, nil + case "<": + return compareResult == -1, nil + case ">": + return compareResult == 1, nil + case "<=": + return compareResult <= 0, nil + case ">=": + return compareResult >= 0, nil + default: + return false, errors.New("unknown operator") + } +} diff --git a/pkg/analyze/ceph_test.go b/pkg/analyze/ceph_test.go new file mode 100644 index 00000000..0f2b5b84 --- /dev/null +++ b/pkg/analyze/ceph_test.go @@ -0,0 +1,204 @@ +package analyzer + +import ( + "fmt" + "testing" + + troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.undefinedlabs.com/scopeagent" +) + +func Test_cephStatus(t *testing.T) { + tests := []struct { + name string + analyzer troubleshootv1beta2.CephStatusAnalyze + expectResult AnalyzeResult + filePath, file string + }{ + { + name: "pass case", + analyzer: troubleshootv1beta2.CephStatusAnalyze{}, + expectResult: AnalyzeResult{ + IsPass: true, + IsWarn: false, + IsFail: false, + Title: "Ceph Status", + Message: "Ceph is healthy", + IconKey: "rook", + IconURI: "https://troubleshoot.sh/images/analyzer-icons/rook.svg?w=11&h=16", + }, + filePath: "ceph/status.json", + file: `{ + "fsid": "96a8178c-6aa2-4adf-a309-9e8869a79611", + "health": { + "status": "HEALTH_OK" + } + }`, + }, + { + name: "warn case", + analyzer: troubleshootv1beta2.CephStatusAnalyze{}, + expectResult: AnalyzeResult{ + IsPass: false, + IsWarn: true, + IsFail: false, + Title: "Ceph Status", + Message: "Ceph status is HEALTH_WARN", + URI: "https://rook.io/docs/rook/v1.4/ceph-common-issues.html", + IconKey: "rook", + IconURI: "https://troubleshoot.sh/images/analyzer-icons/rook.svg?w=11&h=16", + }, + filePath: "ceph/status.json", + file: `{ + "fsid": "96a8178c-6aa2-4adf-a309-9e8869a79611", + "health": { + "status": "HEALTH_WARN" + } + }`, + }, + { + name: "fail case", + analyzer: troubleshootv1beta2.CephStatusAnalyze{}, + expectResult: AnalyzeResult{ + IsPass: false, + IsWarn: false, + IsFail: true, + Title: "Ceph Status", + Message: "Ceph status is HEALTH_ERR", + URI: "https://rook.io/docs/rook/v1.4/ceph-common-issues.html", + IconKey: "rook", + IconURI: "https://troubleshoot.sh/images/analyzer-icons/rook.svg?w=11&h=16", + }, + filePath: "ceph/status.json", + file: `{ + "fsid": "96a8178c-6aa2-4adf-a309-9e8869a79611", + "health": { + "status": "HEALTH_ERR" + } + }`, + }, + { + name: "CollectorName and Namespace", + analyzer: troubleshootv1beta2.CephStatusAnalyze{ + CollectorName: "custom-namespace", + Namespace: "namespace", + }, + expectResult: AnalyzeResult{ + IsPass: true, + IsWarn: false, + IsFail: false, + Title: "Ceph Status", + Message: "Ceph is healthy", + IconKey: "rook", + IconURI: "https://troubleshoot.sh/images/analyzer-icons/rook.svg?w=11&h=16", + }, + filePath: "custom-namespace/namespace/ceph/status.json", + file: `{ + "fsid": "96a8178c-6aa2-4adf-a309-9e8869a79611", + "health": { + "status": "HEALTH_OK" + } + }`, + }, + { + name: "outcomes when", + analyzer: troubleshootv1beta2.CephStatusAnalyze{ + Outcomes: []*troubleshootv1beta2.Outcome{ + { + Fail: &troubleshootv1beta2.SingleOutcome{ + When: "== HEALTH_OK", + Message: "custom message OK", + URI: "custom uri OK", + }, + }, + { + Fail: &troubleshootv1beta2.SingleOutcome{ + When: "<= HEALTH_WARN", + Message: "custom message WARN", + URI: "custom uri WARN", + }, + }, + }, + }, + expectResult: AnalyzeResult{ + IsPass: false, + IsWarn: false, + IsFail: true, + Title: "Ceph Status", + Message: "custom message WARN", + URI: "custom uri WARN", + IconKey: "rook", + IconURI: "https://troubleshoot.sh/images/analyzer-icons/rook.svg?w=11&h=16", + }, + filePath: "ceph/status.json", + file: `{ + "fsid": "96a8178c-6aa2-4adf-a309-9e8869a79611", + "health": { + "status": "HEALTH_WARN" + } + }`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + scopetest := scopeagent.StartTest(t) + defer scopetest.End() + req := require.New(t) + + getFile := func(n string) ([]byte, error) { + assert.Equal(t, n, test.filePath) + return []byte(test.file), nil + } + + actual, err := cephStatus(&test.analyzer, getFile) + req.NoError(err) + + assert.Equal(t, test.expectResult, *actual) + }) + } +} + +func Test_compareCephStatus(t *testing.T) { + tests := []struct { + actual string + when string + want bool + wantErr bool + }{ + { + actual: "HEALTH_OK", + when: "HEALTH_OK", + want: true, + }, + { + actual: "HEALTH_OK", + when: "HEALTH_WARN", + want: false, + }, + { + actual: "HEALTH_OK", + when: "<= HEALTH_WARN", + want: false, + }, + { + actual: "HEALTH_OK", + when: ">= HEALTH_WARN", + want: true, + }, + } + for _, tt := range tests { + t.Run(fmt.Sprintf("%s %s", tt.actual, tt.when), func(t *testing.T) { + got, err := compareCephStatus(tt.actual, tt.when) + if (err != nil) != tt.wantErr { + t.Errorf("compareCephStatus() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("compareCephStatus() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go b/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go index cb608639..a985823a 100644 --- a/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go +++ b/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go @@ -121,6 +121,13 @@ type CollectdAnalyze struct { CollectorName string `json:"collectorName" yaml:"collectorName"` } +type CephStatusAnalyze struct { + AnalyzeMeta `json:",inline" yaml:",inline"` + Outcomes []*Outcome `json:"outcomes" yaml:"outcomes"` + CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"` + Namespace string `json:"namespace" yaml:"namespace"` +} + type AnalyzeMeta struct { CheckName string `json:"checkName,omitempty" yaml:"checkName,omitempty"` Exclude multitype.BoolOrString `json:"exclude,omitempty" yaml:"exclude,omitempty"` @@ -142,4 +149,5 @@ type Analyze struct { Postgres *DatabaseAnalyze `json:"postgres,omitempty" yaml:"postgres,omitempty"` Mysql *DatabaseAnalyze `json:"mysql,omitempty" yaml:"mysql,omitempty"` Redis *DatabaseAnalyze `json:"redis,omitempty" yaml:"redis,omitempty"` + CephStatus *CephStatusAnalyze `json:"cephStatus,omitempty" yaml:"cephStatus,omitempty"` } diff --git a/pkg/apis/troubleshoot/v1beta2/collector_shared.go b/pkg/apis/troubleshoot/v1beta2/collector_shared.go index e1a239b2..d4370a48 100644 --- a/pkg/apis/troubleshoot/v1beta2/collector_shared.go +++ b/pkg/apis/troubleshoot/v1beta2/collector_shared.go @@ -133,6 +133,13 @@ type Collectd struct { HostPath string `json:"hostPath" yaml:"hostPath"` } +type Ceph struct { + CollectorMeta `json:",inline" yaml:",inline"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Namespace string `json:"namespace" yaml:"namespace"` + Timeout string `json:"timeout,omitempty" yaml:"timeout,omitempty"` +} + type Collect struct { ClusterInfo *ClusterInfo `json:"clusterInfo,omitempty" yaml:"clusterInfo,omitempty"` ClusterResources *ClusterResources `json:"clusterResources,omitempty" yaml:"clusterResources,omitempty"` @@ -147,6 +154,7 @@ type Collect struct { Mysql *Database `json:"mysql,omitempty" yaml:"mysql,omitempty"` Redis *Database `json:"redis,omitempty" yaml:"redis,omitempty"` Collectd *Collectd `json:"collectd,omitempty" yaml:"collectd,omitempty"` + Ceph *Ceph `json:"ceph,omitempty" yaml:"ceph,omitempty"` } func (c *Collect) AccessReviewSpecs(overrideNS string) []authorizationv1.SelfSubjectAccessReviewSpec { @@ -350,6 +358,10 @@ func (c *Collect) GetName() string { collector = "collectd" name = c.Collectd.CollectorName } + if c.Ceph != nil { + collector = "ceph" + name = c.Ceph.CollectorName + } if collector == "" { return "" diff --git a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go index f947667f..70de7ae3 100644 --- a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go @@ -127,6 +127,11 @@ func (in *Analyze) DeepCopyInto(out *Analyze) { *out = new(DatabaseAnalyze) (*in).DeepCopyInto(*out) } + if in.CephStatus != nil { + in, out := &in.CephStatus, &out.CephStatus + *out = new(CephStatusAnalyze) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Analyze. @@ -282,6 +287,49 @@ func (in *AnalyzerStatus) DeepCopy() *AnalyzerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Ceph) DeepCopyInto(out *Ceph) { + *out = *in + out.CollectorMeta = in.CollectorMeta +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Ceph. +func (in *Ceph) DeepCopy() *Ceph { + if in == nil { + return nil + } + out := new(Ceph) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CephStatusAnalyze) DeepCopyInto(out *CephStatusAnalyze) { + *out = *in + out.AnalyzeMeta = in.AnalyzeMeta + if in.Outcomes != nil { + in, out := &in.Outcomes, &out.Outcomes + *out = make([]*Outcome, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Outcome) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephStatusAnalyze. +func (in *CephStatusAnalyze) DeepCopy() *CephStatusAnalyze { + if in == nil { + return nil + } + out := new(CephStatusAnalyze) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterInfo) DeepCopyInto(out *ClusterInfo) { *out = *in @@ -409,6 +457,11 @@ func (in *Collect) DeepCopyInto(out *Collect) { *out = new(Collectd) (*in).DeepCopyInto(*out) } + if in.Ceph != nil { + in, out := &in.Ceph, &out.Ceph + *out = new(Ceph) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Collect. diff --git a/pkg/collect/ceph.go b/pkg/collect/ceph.go new file mode 100644 index 00000000..a1dcea3d --- /dev/null +++ b/pkg/collect/ceph.go @@ -0,0 +1,159 @@ +package collect + +import ( + "context" + "fmt" + "path" + "strings" + + multierror "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + DefaultCephNamespace = "rook-ceph" +) + +type CephCommand struct { + ID string + Command []string + Args []string +} + +var CephCommands = []CephCommand{ + { + ID: "status", + Command: []string{"ceph", "status"}, + Args: []string{"-f", "json-pretty"}, + }, + { + ID: "fs", + Command: []string{"ceph", "fs", "status"}, + Args: []string{"-f", "json-pretty"}, + }, + { + ID: "fs-ls", + Command: []string{"ceph", "fs", "ls"}, + Args: []string{"-f", "json-pretty"}, + }, + { + ID: "osd-status", + Command: []string{"ceph", "osd", "status"}, + Args: []string{"-f", "json-pretty"}, + }, + { + ID: "osd-tree", + Command: []string{"ceph", "osd", "tree"}, + Args: []string{"-f", "json-pretty"}, + }, + { + ID: "osd-pool", + Command: []string{"ceph", "osd", "pool", "ls", "detail"}, + Args: []string{"-f", "json-pretty"}, + }, + { + ID: "health", + Command: []string{"ceph", "health", "detail"}, + Args: []string{"-f", "json-pretty"}, + }, + { + ID: "auth", + Command: []string{"ceph", "auth", "ls"}, + Args: []string{"-f", "json-pretty"}, + }, +} + +func Ceph(c *Collector, cephCollector *troubleshootv1beta2.Ceph) (map[string][]byte, error) { + ctx := context.TODO() + + if cephCollector.Namespace == "" { + cephCollector.Namespace = DefaultCephNamespace + } + + pod, err := findRookCephToolsPod(ctx, c, cephCollector.Namespace) + if err != nil { + return nil, err + } + + final := map[string][]byte{} + var multiErr *multierror.Error + for _, command := range CephCommands { + results, err := cephCommandExec(ctx, c, cephCollector, pod, command) + multiErr = multierror.Append(multiErr, errors.Wrapf(err, "failed to exec command %s", command.ID)) + for fileName, output := range results { + final[fileName] = output + } + } + return final, nil +} + +func cephCommandExec(ctx context.Context, c *Collector, cephCollector *troubleshootv1beta2.Ceph, pod *corev1.Pod, command CephCommand) (map[string][]byte, error) { + execCollector := &troubleshootv1beta2.Exec{ + Selector: labelsToSelector(pod.Labels), + Namespace: pod.Namespace, + Command: command.Command, + Args: command.Args, + Timeout: cephCollector.Timeout, + } + results, err := Exec(c, execCollector) + if err != nil { + return nil, err + } + + final := map[string][]byte{} + for filename, result := range results { + pathPrefix := GetCephCollectorFilepath(cephCollector.Name, cephCollector.Namespace) + switch { + case strings.HasSuffix(filename, "-stdout.txt"): + final[path.Join(pathPrefix, fmt.Sprintf("%s.json", command.ID))] = result + case strings.HasSuffix(filename, "-stderr.txt"): + final[path.Join(pathPrefix, fmt.Sprintf("%s-stderr.json", command.ID))] = result + case strings.HasSuffix(filename, "-errors.json"): + final[path.Join(pathPrefix, fmt.Sprintf("%s-errors.json", command.ID))] = result + } + } + + return final, nil +} + +func findRookCephToolsPod(ctx context.Context, c *Collector, namespace string) (*corev1.Pod, error) { + client, err := kubernetes.NewForConfig(c.ClientConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to create kubernetes client") + } + + pods, _ := listPodsInSelectors(ctx, client, namespace, []string{"app=rook-ceph-tools"}) + if len(pods) > 0 { + return &pods[0], nil + } + + pods, _ = listPodsInSelectors(ctx, client, namespace, []string{"app=rook-ceph-operator"}) + if len(pods) > 0 { + return &pods[0], nil + } + + return nil, errors.New("rook ceph tools pod not found") +} + +func GetCephCollectorFilepath(name, namespace string) string { + parts := []string{} + if name != "" { + parts = append(parts, name) + } + if namespace != "" && namespace != DefaultCephNamespace { + parts = append(parts, namespace) + } + parts = append(parts, "ceph") + return path.Join(parts...) +} + +func labelsToSelector(labels map[string]string) []string { + selector := []string{} + for key, value := range labels { + selector = append(selector, fmt.Sprintf("%s=%s", key, value)) + } + return selector +} diff --git a/pkg/collect/collector.go b/pkg/collect/collector.go index bd73a8ba..df985d4b 100644 --- a/pkg/collect/collector.go +++ b/pkg/collect/collector.go @@ -168,6 +168,15 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta2.Reda return } result, err = Collectd(c, c.Collect.Collectd) + } else if c.Collect.Ceph != nil { + isExcludedResult, err = isExcluded(c.Collect.Ceph.Exclude) + if err != nil { + return nil, err + } + if isExcludedResult { + return nil, nil + } + result, err = Ceph(c, c.Collect.Ceph) } else { err = errors.New("no spec found to run") return