mirror of
https://github.com/replicatedhq/troubleshoot.git
synced 2026-04-15 07:16:34 +00:00
282 lines
9.2 KiB
Go
282 lines
9.2 KiB
Go
package analyzer
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
"github.com/replicatedhq/troubleshoot/internal/util"
|
|
troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
|
|
"github.com/replicatedhq/troubleshoot/pkg/collect"
|
|
)
|
|
|
|
type AnalyzeHostFilesystemPerformance struct {
|
|
hostAnalyzer *troubleshootv1beta2.FilesystemPerformanceAnalyze
|
|
}
|
|
|
|
func (a *AnalyzeHostFilesystemPerformance) Title() string {
|
|
return hostAnalyzerTitleOrDefault(a.hostAnalyzer.AnalyzeMeta, "Filesystem Performance")
|
|
}
|
|
|
|
func (a *AnalyzeHostFilesystemPerformance) IsExcluded() (bool, error) {
|
|
return isExcluded(a.hostAnalyzer.Exclude)
|
|
}
|
|
|
|
func (a *AnalyzeHostFilesystemPerformance) Analyze(
|
|
getCollectedFileContents func(string) ([]byte, error), findFiles getChildCollectedFileContents,
|
|
) ([]*AnalyzeResult, error) {
|
|
result := &AnalyzeResult{Title: a.Title()}
|
|
hostAnalyzer := a.hostAnalyzer
|
|
collectorName := a.hostAnalyzer.CollectorName
|
|
|
|
if collectorName == "" {
|
|
collectorName = "filesystemPerformance"
|
|
}
|
|
const nodeBaseDir = "host-collectors/filesystemPerformance"
|
|
localPath := fmt.Sprintf("%s/%s.json", nodeBaseDir, collectorName)
|
|
fileName := fmt.Sprintf("%s.json", collectorName)
|
|
|
|
collectedContents, err := retrieveCollectedContents(
|
|
getCollectedFileContents,
|
|
localPath,
|
|
nodeBaseDir,
|
|
fileName,
|
|
)
|
|
if err != nil {
|
|
if len(hostAnalyzer.Outcomes) >= 1 {
|
|
// if the very first outcome is FILE_NOT_COLLECTED', then use that outcome
|
|
// otherwise, return the error
|
|
if hostAnalyzer.Outcomes[0].Fail != nil && hostAnalyzer.Outcomes[0].Fail.When == FILE_NOT_COLLECTED {
|
|
result.IsFail = true
|
|
result.Message = renderFSPerfOutcome(hostAnalyzer.Outcomes[0].Fail.Message, collect.FSPerfResults{})
|
|
result.URI = hostAnalyzer.Outcomes[0].Fail.URI
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if hostAnalyzer.Outcomes[0].Warn != nil && hostAnalyzer.Outcomes[0].Warn.When == FILE_NOT_COLLECTED {
|
|
result.IsWarn = true
|
|
result.Message = renderFSPerfOutcome(hostAnalyzer.Outcomes[0].Warn.Message, collect.FSPerfResults{})
|
|
result.URI = hostAnalyzer.Outcomes[0].Warn.URI
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
if hostAnalyzer.Outcomes[0].Pass != nil && hostAnalyzer.Outcomes[0].Pass.When == FILE_NOT_COLLECTED {
|
|
result.IsPass = true
|
|
result.Message = renderFSPerfOutcome(hostAnalyzer.Outcomes[0].Pass.Message, collect.FSPerfResults{})
|
|
result.URI = hostAnalyzer.Outcomes[0].Pass.URI
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
return nil, errors.Wrapf(err, "failed to get collected file %s", localPath)
|
|
}
|
|
}
|
|
|
|
var results []*AnalyzeResult
|
|
for _, content := range collectedContents {
|
|
currentTitle := a.Title()
|
|
if content.NodeName != "" {
|
|
currentTitle = fmt.Sprintf("%s - Node %s", a.Title(), content.NodeName)
|
|
}
|
|
result, err := a.analyzeSingleNode(content, currentTitle)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed to analyze filesystem performance for %s", currentTitle)
|
|
}
|
|
if result != nil {
|
|
results = append(results, result...)
|
|
}
|
|
}
|
|
|
|
return results, nil
|
|
}
|
|
|
|
func compareHostFilesystemPerformanceConditionalToActual(conditional string, fsPerf collect.FSPerfResults) (res bool, err error) {
|
|
if conditional == FILE_NOT_COLLECTED {
|
|
return false, nil
|
|
}
|
|
|
|
parts := strings.Split(conditional, " ")
|
|
if len(parts) != 3 {
|
|
return false, fmt.Errorf("conditional must have exactly 3 parts, got %d", len(parts))
|
|
}
|
|
keyword := strings.ToLower(parts[0])
|
|
comparator := parts[1]
|
|
|
|
desiredDuration, err := time.ParseDuration(parts[2])
|
|
if err != nil {
|
|
return false, errors.Wrapf(err, "failed to parse duration %q", parts[2])
|
|
}
|
|
|
|
switch keyword {
|
|
case "min":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.Min, desiredDuration)
|
|
case "max":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.Max, desiredDuration)
|
|
case "average":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.Average, desiredDuration)
|
|
case "p1":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P1, desiredDuration)
|
|
case "p5":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P5, desiredDuration)
|
|
case "p10":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P10, desiredDuration)
|
|
case "p20":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P20, desiredDuration)
|
|
case "p30":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P30, desiredDuration)
|
|
case "p40":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P40, desiredDuration)
|
|
case "p50":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P50, desiredDuration)
|
|
case "p60":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P60, desiredDuration)
|
|
case "p70":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P70, desiredDuration)
|
|
case "p80":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P80, desiredDuration)
|
|
case "p90":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P90, desiredDuration)
|
|
case "p95":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P95, desiredDuration)
|
|
case "p99":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P99, desiredDuration)
|
|
case "p995":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P995, desiredDuration)
|
|
case "p999":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P999, desiredDuration)
|
|
case "p9995":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P9995, desiredDuration)
|
|
case "p9999":
|
|
return doCompareHostFilesystemPerformance(comparator, fsPerf.P9999, desiredDuration)
|
|
}
|
|
|
|
return false, fmt.Errorf("unknown filesystem performance keyword %q", keyword)
|
|
}
|
|
|
|
func doCompareHostFilesystemPerformance(operator string, actual time.Duration, desired time.Duration) (bool, error) {
|
|
switch operator {
|
|
case "<":
|
|
return actual < desired, nil
|
|
case "<=":
|
|
return actual <= desired, nil
|
|
case ">":
|
|
return actual > desired, nil
|
|
case ">=":
|
|
return actual >= desired, nil
|
|
case "=", "==", "===":
|
|
return actual == desired, nil
|
|
}
|
|
|
|
return false, fmt.Errorf("unknown filesystem performance operator %q", operator)
|
|
}
|
|
|
|
func renderFSPerfOutcome(outcome string, fsPerf collect.FSPerfResults) string {
|
|
rendered, err := util.RenderTemplate(outcome, fsPerf)
|
|
if err != nil {
|
|
log.Printf("Failed to render filesystem performance outcome: %v", err)
|
|
return outcome
|
|
}
|
|
return rendered
|
|
}
|
|
|
|
func (a *AnalyzeHostFilesystemPerformance) analyzeSingleNode(content collectedContent, currentTitle string) ([]*AnalyzeResult, error) {
|
|
hostAnalyzer := a.hostAnalyzer
|
|
result := &AnalyzeResult{
|
|
Title: currentTitle,
|
|
}
|
|
fioResult := collect.FioResult{}
|
|
if err := json.Unmarshal(content.Data, &fioResult); err != nil {
|
|
return nil, errors.Wrapf(err, "failed to unmarshal fio results from %s", currentTitle)
|
|
}
|
|
|
|
var job *collect.FioJobs
|
|
for _, j := range fioResult.Jobs {
|
|
if j.JobName == collect.FioJobName {
|
|
job = &j
|
|
break
|
|
}
|
|
}
|
|
if job == nil {
|
|
return nil, errors.Errorf("no job named 'fsperf' found in fio results from %s", currentTitle)
|
|
}
|
|
|
|
fioWriteLatency := job.Sync
|
|
|
|
fsPerf := fioWriteLatency.FSPerfResults()
|
|
if err := json.Unmarshal(content.Data, &fsPerf); err != nil {
|
|
return nil, errors.Wrapf(err, "failed to unmarshal filesystem performance results from %s", currentTitle)
|
|
}
|
|
|
|
for _, outcome := range hostAnalyzer.Outcomes {
|
|
|
|
switch {
|
|
case outcome.Fail != nil:
|
|
if outcome.Fail.When == "" {
|
|
result.IsFail = true
|
|
result.Message = renderFSPerfOutcome(outcome.Fail.Message, fsPerf)
|
|
result.URI = outcome.Fail.URI
|
|
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
|
|
isMatch, err := compareHostFilesystemPerformanceConditionalToActual(outcome.Fail.When, fsPerf)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed to compare %q", outcome.Fail.When)
|
|
}
|
|
|
|
if isMatch {
|
|
result.IsFail = true
|
|
result.Message = renderFSPerfOutcome(outcome.Fail.Message, fsPerf)
|
|
result.URI = outcome.Fail.URI
|
|
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
case outcome.Warn != nil:
|
|
if outcome.Warn.When == "" {
|
|
result.IsWarn = true
|
|
result.Message = renderFSPerfOutcome(outcome.Warn.Message, fsPerf)
|
|
result.URI = outcome.Warn.URI
|
|
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
|
|
isMatch, err := compareHostFilesystemPerformanceConditionalToActual(outcome.Warn.When, fsPerf)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed to compare %q", outcome.Warn.When)
|
|
}
|
|
|
|
if isMatch {
|
|
result.IsWarn = true
|
|
result.Message = renderFSPerfOutcome(outcome.Warn.Message, fsPerf)
|
|
result.URI = outcome.Warn.URI
|
|
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
case outcome.Pass != nil:
|
|
if outcome.Pass.When == "" {
|
|
result.IsPass = true
|
|
result.Message = renderFSPerfOutcome(outcome.Pass.Message, fsPerf)
|
|
result.URI = outcome.Pass.URI
|
|
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
|
|
isMatch, err := compareHostFilesystemPerformanceConditionalToActual(outcome.Pass.When, fsPerf)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed to compare %q", outcome.Pass.When)
|
|
}
|
|
|
|
if isMatch {
|
|
result.IsPass = true
|
|
result.Message = renderFSPerfOutcome(outcome.Pass.Message, fsPerf)
|
|
result.URI = outcome.Pass.URI
|
|
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|
|
default:
|
|
return nil, errors.New("unexpected outcome")
|
|
}
|
|
}
|
|
|
|
return []*AnalyzeResult{result}, nil
|
|
}
|