mirror of
https://github.com/stakater/Reloader.git
synced 2026-02-14 18:09:50 +00:00
861 lines
29 KiB
Go
861 lines
29 KiB
Go
package cmd
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"log"
|
||
"math"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/spf13/cobra"
|
||
)
|
||
|
||
var (
|
||
reportScenario string
|
||
reportResultsDir string
|
||
reportOutputFile string
|
||
reportFormat string
|
||
)
|
||
|
||
var reportCmd = &cobra.Command{
|
||
Use: "report",
|
||
Short: "Generate comparison report for a scenario",
|
||
Long: `Generate a detailed report for a specific test scenario.
|
||
|
||
Examples:
|
||
# Generate report for a scenario
|
||
loadtest report --scenario=S2 --results-dir=./results
|
||
|
||
# Generate JSON report
|
||
loadtest report --scenario=S2 --format=json`,
|
||
Run: func(cmd *cobra.Command, args []string) {
|
||
reportCommand()
|
||
},
|
||
}
|
||
|
||
func init() {
|
||
reportCmd.Flags().StringVar(&reportScenario, "scenario", "", "Scenario to report on (required)")
|
||
reportCmd.Flags().StringVar(&reportResultsDir, "results-dir", "./results", "Directory containing results")
|
||
reportCmd.Flags().StringVar(&reportOutputFile, "output", "", "Output file (default: stdout)")
|
||
reportCmd.Flags().StringVar(&reportFormat, "format", "text", "Output format: text, json, markdown")
|
||
reportCmd.MarkFlagRequired("scenario")
|
||
}
|
||
|
||
// PrometheusResponse represents a Prometheus API response for report parsing.
|
||
type PrometheusResponse struct {
|
||
Status string `json:"status"`
|
||
Data struct {
|
||
ResultType string `json:"resultType"`
|
||
Result []struct {
|
||
Metric map[string]string `json:"metric"`
|
||
Value []interface{} `json:"value"`
|
||
} `json:"result"`
|
||
} `json:"data"`
|
||
}
|
||
|
||
// MetricComparison represents the comparison of a single metric.
|
||
type MetricComparison struct {
|
||
Name string `json:"name"`
|
||
DisplayName string `json:"display_name"`
|
||
Unit string `json:"unit"`
|
||
IsCounter bool `json:"is_counter"`
|
||
OldValue float64 `json:"old_value"`
|
||
NewValue float64 `json:"new_value"`
|
||
Expected float64 `json:"expected"`
|
||
Difference float64 `json:"difference"`
|
||
DiffPct float64 `json:"diff_pct"`
|
||
Status string `json:"status"`
|
||
Threshold float64 `json:"threshold"`
|
||
OldMeetsExpected string `json:"old_meets_expected"`
|
||
NewMeetsExpected string `json:"new_meets_expected"`
|
||
}
|
||
|
||
type metricInfo struct {
|
||
unit string
|
||
isCounter bool
|
||
}
|
||
|
||
var metricInfoMap = map[string]metricInfo{
|
||
"reconcile_total": {unit: "count", isCounter: true},
|
||
"reconcile_duration_p50": {unit: "s", isCounter: false},
|
||
"reconcile_duration_p95": {unit: "s", isCounter: false},
|
||
"reconcile_duration_p99": {unit: "s", isCounter: false},
|
||
"action_total": {unit: "count", isCounter: true},
|
||
"action_latency_p50": {unit: "s", isCounter: false},
|
||
"action_latency_p95": {unit: "s", isCounter: false},
|
||
"action_latency_p99": {unit: "s", isCounter: false},
|
||
"errors_total": {unit: "count", isCounter: true},
|
||
"reload_executed_total": {unit: "count", isCounter: true},
|
||
"workloads_scanned_total": {unit: "count", isCounter: true},
|
||
"workloads_matched_total": {unit: "count", isCounter: true},
|
||
"skipped_total_no_data_change": {unit: "count", isCounter: true},
|
||
"rest_client_requests_total": {unit: "count", isCounter: true},
|
||
"rest_client_requests_get": {unit: "count", isCounter: true},
|
||
"rest_client_requests_patch": {unit: "count", isCounter: true},
|
||
"rest_client_requests_put": {unit: "count", isCounter: true},
|
||
"rest_client_requests_errors": {unit: "count", isCounter: true},
|
||
"memory_rss_mb_avg": {unit: "MB", isCounter: false},
|
||
"memory_rss_mb_max": {unit: "MB", isCounter: false},
|
||
"memory_heap_mb_avg": {unit: "MB", isCounter: false},
|
||
"memory_heap_mb_max": {unit: "MB", isCounter: false},
|
||
"cpu_cores_avg": {unit: "cores", isCounter: false},
|
||
"cpu_cores_max": {unit: "cores", isCounter: false},
|
||
"goroutines_avg": {unit: "count", isCounter: false},
|
||
"goroutines_max": {unit: "count", isCounter: false},
|
||
"gc_pause_p99_ms": {unit: "ms", isCounter: false},
|
||
}
|
||
|
||
// ReportExpectedMetrics matches the expected metrics from test scenarios.
|
||
type ReportExpectedMetrics struct {
|
||
ActionTotal int `json:"action_total"`
|
||
ReloadExecutedTotal int `json:"reload_executed_total"`
|
||
ReconcileTotal int `json:"reconcile_total"`
|
||
WorkloadsScannedTotal int `json:"workloads_scanned_total"`
|
||
WorkloadsMatchedTotal int `json:"workloads_matched_total"`
|
||
SkippedTotal int `json:"skipped_total"`
|
||
Description string `json:"description"`
|
||
}
|
||
|
||
// ScenarioReport represents the full report for a scenario.
|
||
type ScenarioReport struct {
|
||
Scenario string `json:"scenario"`
|
||
Timestamp time.Time `json:"timestamp"`
|
||
Comparisons []MetricComparison `json:"comparisons"`
|
||
OverallStatus string `json:"overall_status"`
|
||
Summary string `json:"summary"`
|
||
PassCriteria []string `json:"pass_criteria"`
|
||
FailedCriteria []string `json:"failed_criteria"`
|
||
Expected ReportExpectedMetrics `json:"expected"`
|
||
TestDescription string `json:"test_description"`
|
||
}
|
||
|
||
// MetricType defines how to evaluate a metric.
|
||
type MetricType int
|
||
|
||
const (
|
||
LowerIsBetter MetricType = iota
|
||
ShouldMatch
|
||
HigherIsBetter
|
||
Informational
|
||
)
|
||
|
||
type thresholdConfig struct {
|
||
maxDiff float64
|
||
metricType MetricType
|
||
minAbsDiff float64
|
||
}
|
||
|
||
var thresholds = map[string]thresholdConfig{
|
||
"reconcile_total": {maxDiff: 60.0, metricType: LowerIsBetter},
|
||
"reconcile_duration_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5},
|
||
"reconcile_duration_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0},
|
||
"reconcile_duration_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0},
|
||
"action_latency_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5},
|
||
"action_latency_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0},
|
||
"action_latency_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0},
|
||
"errors_total": {maxDiff: 0.0, metricType: LowerIsBetter},
|
||
"action_total": {maxDiff: 15.0, metricType: ShouldMatch},
|
||
"reload_executed_total": {maxDiff: 15.0, metricType: ShouldMatch},
|
||
"workloads_scanned_total": {maxDiff: 15.0, metricType: ShouldMatch},
|
||
"workloads_matched_total": {maxDiff: 15.0, metricType: ShouldMatch},
|
||
"skipped_total_no_data_change": {maxDiff: 20.0, metricType: ShouldMatch},
|
||
"rest_client_requests_total": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50},
|
||
"rest_client_requests_get": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50},
|
||
"rest_client_requests_patch": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50},
|
||
"rest_client_requests_put": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 20},
|
||
"rest_client_requests_errors": {maxDiff: 0.0, metricType: LowerIsBetter, minAbsDiff: 100},
|
||
"memory_rss_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20},
|
||
"memory_rss_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 30},
|
||
"memory_heap_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 15},
|
||
"memory_heap_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20},
|
||
"cpu_cores_avg": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.1},
|
||
"cpu_cores_max": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.2},
|
||
"goroutines_avg": {metricType: Informational},
|
||
"goroutines_max": {metricType: Informational},
|
||
"gc_pause_p99_ms": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 5},
|
||
}
|
||
|
||
func reportCommand() {
|
||
if reportScenario == "" {
|
||
log.Fatal("--scenario is required for report command")
|
||
}
|
||
|
||
report, err := generateScenarioReport(reportScenario, reportResultsDir)
|
||
if err != nil {
|
||
log.Fatalf("Failed to generate report: %v", err)
|
||
}
|
||
|
||
var output string
|
||
switch OutputFormat(reportFormat) {
|
||
case OutputFormatJSON:
|
||
output = renderScenarioReportJSON(report)
|
||
case OutputFormatMarkdown:
|
||
output = renderScenarioReportMarkdown(report)
|
||
default:
|
||
output = renderScenarioReport(report)
|
||
}
|
||
|
||
if reportOutputFile != "" {
|
||
if err := os.WriteFile(reportOutputFile, []byte(output), 0644); err != nil {
|
||
log.Fatalf("Failed to write output file: %v", err)
|
||
}
|
||
log.Printf("Report written to %s", reportOutputFile)
|
||
} else {
|
||
fmt.Println(output)
|
||
}
|
||
}
|
||
|
||
func generateScenarioReport(scenario, resultsDir string) (*ScenarioReport, error) {
|
||
oldDir := filepath.Join(resultsDir, scenario, "old")
|
||
newDir := filepath.Join(resultsDir, scenario, "new")
|
||
scenarioDir := filepath.Join(resultsDir, scenario)
|
||
|
||
_, oldErr := os.Stat(oldDir)
|
||
_, newErr := os.Stat(newDir)
|
||
hasOld := oldErr == nil
|
||
hasNew := newErr == nil
|
||
isComparison := hasOld && hasNew
|
||
|
||
singleVersion := ""
|
||
singleDir := ""
|
||
if !isComparison {
|
||
if hasNew {
|
||
singleVersion = "new"
|
||
singleDir = newDir
|
||
} else if hasOld {
|
||
singleVersion = "old"
|
||
singleDir = oldDir
|
||
} else {
|
||
return nil, fmt.Errorf("no results found in %s", scenarioDir)
|
||
}
|
||
}
|
||
|
||
report := &ScenarioReport{
|
||
Scenario: scenario,
|
||
Timestamp: time.Now(),
|
||
}
|
||
|
||
expectedPath := filepath.Join(scenarioDir, "expected.json")
|
||
if data, err := os.ReadFile(expectedPath); err == nil {
|
||
if err := json.Unmarshal(data, &report.Expected); err != nil {
|
||
log.Printf("Warning: Could not parse expected metrics: %v", err)
|
||
} else {
|
||
report.TestDescription = report.Expected.Description
|
||
}
|
||
}
|
||
|
||
if !isComparison {
|
||
return generateSingleVersionReport(report, singleDir, singleVersion, scenario)
|
||
}
|
||
|
||
metricsToCompare := []struct {
|
||
name string
|
||
file string
|
||
selector func(data PrometheusResponse) float64
|
||
}{
|
||
{"reconcile_total", "reloader_reconcile_total.json", sumAllValues},
|
||
{"reconcile_duration_p50", "reconcile_p50.json", getFirstValue},
|
||
{"reconcile_duration_p95", "reconcile_p95.json", getFirstValue},
|
||
{"reconcile_duration_p99", "reconcile_p99.json", getFirstValue},
|
||
{"action_total", "reloader_action_total.json", sumAllValues},
|
||
{"action_latency_p50", "action_p50.json", getFirstValue},
|
||
{"action_latency_p95", "action_p95.json", getFirstValue},
|
||
{"action_latency_p99", "action_p99.json", getFirstValue},
|
||
{"errors_total", "reloader_errors_total.json", sumAllValues},
|
||
{"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues},
|
||
{"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues},
|
||
{"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues},
|
||
{"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue},
|
||
{"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue},
|
||
{"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue},
|
||
{"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue},
|
||
{"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue},
|
||
{"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB},
|
||
{"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB},
|
||
{"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB},
|
||
{"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB},
|
||
{"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue},
|
||
{"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue},
|
||
{"goroutines_avg", "goroutines_avg.json", getFirstValue},
|
||
{"goroutines_max", "goroutines_max.json", getFirstValue},
|
||
{"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs},
|
||
}
|
||
|
||
expectedValues := map[string]float64{
|
||
"action_total": float64(report.Expected.ActionTotal),
|
||
"reload_executed_total": float64(report.Expected.ReloadExecutedTotal),
|
||
"reconcile_total": float64(report.Expected.ReconcileTotal),
|
||
"workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal),
|
||
"workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal),
|
||
"skipped_total": float64(report.Expected.SkippedTotal),
|
||
}
|
||
|
||
metricValues := make(map[string]struct{ old, new, expected float64 })
|
||
|
||
for _, m := range metricsToCompare {
|
||
oldData, err := loadMetricFile(filepath.Join(oldDir, m.file))
|
||
if err != nil {
|
||
log.Printf("Warning: Could not load old metric %s: %v", m.name, err)
|
||
continue
|
||
}
|
||
|
||
newData, err := loadMetricFile(filepath.Join(newDir, m.file))
|
||
if err != nil {
|
||
log.Printf("Warning: Could not load new metric %s: %v", m.name, err)
|
||
continue
|
||
}
|
||
|
||
oldValue := m.selector(oldData)
|
||
newValue := m.selector(newData)
|
||
expected := expectedValues[m.name]
|
||
|
||
metricValues[m.name] = struct{ old, new, expected float64 }{oldValue, newValue, expected}
|
||
}
|
||
|
||
newMeetsActionExpected := false
|
||
newReconcileIsZero := false
|
||
isChurnScenario := scenario == "S5"
|
||
if v, ok := metricValues["action_total"]; ok && v.expected > 0 {
|
||
tolerance := v.expected * 0.15
|
||
newMeetsActionExpected = math.Abs(v.new-v.expected) <= tolerance
|
||
}
|
||
if v, ok := metricValues["reconcile_total"]; ok {
|
||
newReconcileIsZero = v.new == 0
|
||
}
|
||
|
||
for _, m := range metricsToCompare {
|
||
v, ok := metricValues[m.name]
|
||
if !ok {
|
||
continue
|
||
}
|
||
|
||
comparison := compareMetricWithExpected(m.name, v.old, v.new, v.expected)
|
||
|
||
if strings.HasPrefix(m.name, "rest_client_requests") {
|
||
if newMeetsActionExpected && comparison.Status != "pass" {
|
||
if oldMeets, ok := metricValues["action_total"]; ok {
|
||
oldTolerance := oldMeets.expected * 0.15
|
||
oldMissed := math.Abs(oldMeets.old-oldMeets.expected) > oldTolerance
|
||
if oldMissed {
|
||
comparison.Status = "pass"
|
||
}
|
||
}
|
||
}
|
||
if newReconcileIsZero && comparison.Status != "pass" {
|
||
comparison.Status = "pass"
|
||
}
|
||
}
|
||
|
||
if isChurnScenario {
|
||
if m.name == "errors_total" {
|
||
if v.new < 50 && v.old < 50 {
|
||
comparison.Status = "pass"
|
||
} else if v.new <= v.old*1.5 {
|
||
comparison.Status = "pass"
|
||
}
|
||
}
|
||
if m.name == "action_total" || m.name == "reload_executed_total" {
|
||
if v.old > 0 {
|
||
diff := math.Abs(v.new-v.old) / v.old * 100
|
||
if diff <= 20 {
|
||
comparison.Status = "pass"
|
||
}
|
||
} else if v.new > 0 {
|
||
comparison.Status = "pass"
|
||
}
|
||
}
|
||
}
|
||
|
||
report.Comparisons = append(report.Comparisons, comparison)
|
||
|
||
if comparison.Status == "pass" {
|
||
report.PassCriteria = append(report.PassCriteria, m.name)
|
||
} else if comparison.Status == "fail" {
|
||
report.FailedCriteria = append(report.FailedCriteria, m.name)
|
||
}
|
||
}
|
||
|
||
if len(report.FailedCriteria) == 0 {
|
||
report.OverallStatus = "PASS"
|
||
report.Summary = "All metrics within acceptable thresholds"
|
||
} else {
|
||
report.OverallStatus = "FAIL"
|
||
report.Summary = fmt.Sprintf("%d metrics failed: %s",
|
||
len(report.FailedCriteria),
|
||
strings.Join(report.FailedCriteria, ", "))
|
||
}
|
||
|
||
return report, nil
|
||
}
|
||
|
||
func generateSingleVersionReport(report *ScenarioReport, dataDir, version, scenario string) (*ScenarioReport, error) {
|
||
metricsToCollect := []struct {
|
||
name string
|
||
file string
|
||
selector func(data PrometheusResponse) float64
|
||
}{
|
||
{"reconcile_total", "reloader_reconcile_total.json", sumAllValues},
|
||
{"reconcile_duration_p50", "reconcile_p50.json", getFirstValue},
|
||
{"reconcile_duration_p95", "reconcile_p95.json", getFirstValue},
|
||
{"reconcile_duration_p99", "reconcile_p99.json", getFirstValue},
|
||
{"action_total", "reloader_action_total.json", sumAllValues},
|
||
{"action_latency_p50", "action_p50.json", getFirstValue},
|
||
{"action_latency_p95", "action_p95.json", getFirstValue},
|
||
{"action_latency_p99", "action_p99.json", getFirstValue},
|
||
{"errors_total", "reloader_errors_total.json", sumAllValues},
|
||
{"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues},
|
||
{"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues},
|
||
{"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues},
|
||
{"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue},
|
||
{"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue},
|
||
{"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue},
|
||
{"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue},
|
||
{"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue},
|
||
{"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB},
|
||
{"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB},
|
||
{"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB},
|
||
{"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB},
|
||
{"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue},
|
||
{"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue},
|
||
{"goroutines_avg", "goroutines_avg.json", getFirstValue},
|
||
{"goroutines_max", "goroutines_max.json", getFirstValue},
|
||
{"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs},
|
||
}
|
||
|
||
expectedValues := map[string]float64{
|
||
"action_total": float64(report.Expected.ActionTotal),
|
||
"reload_executed_total": float64(report.Expected.ReloadExecutedTotal),
|
||
"reconcile_total": float64(report.Expected.ReconcileTotal),
|
||
"workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal),
|
||
"workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal),
|
||
"skipped_total": float64(report.Expected.SkippedTotal),
|
||
}
|
||
|
||
for _, m := range metricsToCollect {
|
||
data, err := loadMetricFile(filepath.Join(dataDir, m.file))
|
||
if err != nil {
|
||
log.Printf("Warning: Could not load metric %s: %v", m.name, err)
|
||
continue
|
||
}
|
||
|
||
value := m.selector(data)
|
||
expected := expectedValues[m.name]
|
||
|
||
info := metricInfoMap[m.name]
|
||
if info.unit == "" {
|
||
info = metricInfo{unit: "count", isCounter: true}
|
||
}
|
||
|
||
displayName := m.name
|
||
if info.unit != "count" {
|
||
displayName = fmt.Sprintf("%s (%s)", m.name, info.unit)
|
||
}
|
||
|
||
status := "info"
|
||
meetsExp := "-"
|
||
|
||
if expected > 0 {
|
||
meetsExp = meetsExpected(value, expected)
|
||
threshold, ok := thresholds[m.name]
|
||
if ok && threshold.metricType == ShouldMatch {
|
||
if meetsExp == "✓" {
|
||
status = "pass"
|
||
report.PassCriteria = append(report.PassCriteria, m.name)
|
||
} else {
|
||
status = "fail"
|
||
report.FailedCriteria = append(report.FailedCriteria, m.name)
|
||
}
|
||
}
|
||
}
|
||
|
||
if info.isCounter {
|
||
value = math.Round(value)
|
||
}
|
||
|
||
report.Comparisons = append(report.Comparisons, MetricComparison{
|
||
Name: m.name,
|
||
DisplayName: displayName,
|
||
Unit: info.unit,
|
||
IsCounter: info.isCounter,
|
||
OldValue: 0,
|
||
NewValue: value,
|
||
Expected: expected,
|
||
OldMeetsExpected: "-",
|
||
NewMeetsExpected: meetsExp,
|
||
Status: status,
|
||
})
|
||
}
|
||
|
||
if len(report.FailedCriteria) == 0 {
|
||
report.OverallStatus = "PASS"
|
||
report.Summary = fmt.Sprintf("Single-version test (%s) completed successfully", version)
|
||
} else {
|
||
report.OverallStatus = "FAIL"
|
||
report.Summary = fmt.Sprintf("%d metrics failed: %s",
|
||
len(report.FailedCriteria),
|
||
strings.Join(report.FailedCriteria, ", "))
|
||
}
|
||
|
||
return report, nil
|
||
}
|
||
|
||
func loadMetricFile(path string) (PrometheusResponse, error) {
|
||
var resp PrometheusResponse
|
||
data, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return resp, err
|
||
}
|
||
err = json.Unmarshal(data, &resp)
|
||
return resp, err
|
||
}
|
||
|
||
func sumAllValues(data PrometheusResponse) float64 {
|
||
var sum float64
|
||
for _, result := range data.Data.Result {
|
||
if len(result.Value) >= 2 {
|
||
if v, ok := result.Value[1].(string); ok {
|
||
var f float64
|
||
fmt.Sscanf(v, "%f", &f)
|
||
sum += f
|
||
}
|
||
}
|
||
}
|
||
return sum
|
||
}
|
||
|
||
func sumSuccessValues(data PrometheusResponse) float64 {
|
||
var sum float64
|
||
for _, result := range data.Data.Result {
|
||
if result.Metric["success"] == "true" {
|
||
if len(result.Value) >= 2 {
|
||
if v, ok := result.Value[1].(string); ok {
|
||
var f float64
|
||
fmt.Sscanf(v, "%f", &f)
|
||
sum += f
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return sum
|
||
}
|
||
|
||
func getFirstValue(data PrometheusResponse) float64 {
|
||
if len(data.Data.Result) > 0 && len(data.Data.Result[0].Value) >= 2 {
|
||
if v, ok := data.Data.Result[0].Value[1].(string); ok {
|
||
var f float64
|
||
fmt.Sscanf(v, "%f", &f)
|
||
return f
|
||
}
|
||
}
|
||
return 0
|
||
}
|
||
|
||
func bytesToMB(data PrometheusResponse) float64 {
|
||
bytes := getFirstValue(data)
|
||
return bytes / (1024 * 1024)
|
||
}
|
||
|
||
func secondsToMs(data PrometheusResponse) float64 {
|
||
seconds := getFirstValue(data)
|
||
return seconds * 1000
|
||
}
|
||
|
||
func meetsExpected(value, expected float64) string {
|
||
if expected == 0 {
|
||
return "-"
|
||
}
|
||
tolerance := expected * 0.15
|
||
if math.Abs(value-expected) <= tolerance {
|
||
return "✓"
|
||
}
|
||
return "✗"
|
||
}
|
||
|
||
func compareMetricWithExpected(name string, oldValue, newValue, expected float64) MetricComparison {
|
||
diff := newValue - oldValue
|
||
absDiff := math.Abs(diff)
|
||
var diffPct float64
|
||
if oldValue != 0 {
|
||
diffPct = (diff / oldValue) * 100
|
||
} else if newValue != 0 {
|
||
diffPct = 100
|
||
}
|
||
|
||
threshold, ok := thresholds[name]
|
||
if !ok {
|
||
threshold = thresholdConfig{maxDiff: 10.0, metricType: ShouldMatch}
|
||
}
|
||
|
||
info := metricInfoMap[name]
|
||
if info.unit == "" {
|
||
info = metricInfo{unit: "count", isCounter: true}
|
||
}
|
||
displayName := name
|
||
if info.unit != "count" {
|
||
displayName = fmt.Sprintf("%s (%s)", name, info.unit)
|
||
}
|
||
|
||
if info.isCounter {
|
||
oldValue = math.Round(oldValue)
|
||
newValue = math.Round(newValue)
|
||
}
|
||
|
||
status := "pass"
|
||
oldMeetsExp := meetsExpected(oldValue, expected)
|
||
newMeetsExp := meetsExpected(newValue, expected)
|
||
|
||
isNewMetric := info.isCounter && oldValue == 0 && newValue > 0 && expected == 0
|
||
|
||
if isNewMetric {
|
||
status = "info"
|
||
} else if expected > 0 && threshold.metricType == ShouldMatch {
|
||
if newMeetsExp == "✗" {
|
||
status = "fail"
|
||
}
|
||
} else {
|
||
switch threshold.metricType {
|
||
case LowerIsBetter:
|
||
if threshold.minAbsDiff > 0 && absDiff < threshold.minAbsDiff {
|
||
status = "pass"
|
||
} else if diffPct > threshold.maxDiff {
|
||
status = "fail"
|
||
}
|
||
case HigherIsBetter:
|
||
if diffPct < -threshold.maxDiff {
|
||
status = "fail"
|
||
}
|
||
case ShouldMatch:
|
||
if math.Abs(diffPct) > threshold.maxDiff {
|
||
status = "fail"
|
||
}
|
||
case Informational:
|
||
status = "info"
|
||
}
|
||
}
|
||
|
||
return MetricComparison{
|
||
Name: name,
|
||
DisplayName: displayName,
|
||
Unit: info.unit,
|
||
IsCounter: info.isCounter,
|
||
Expected: expected,
|
||
OldMeetsExpected: oldMeetsExp,
|
||
NewMeetsExpected: newMeetsExp,
|
||
OldValue: oldValue,
|
||
NewValue: newValue,
|
||
Difference: diff,
|
||
DiffPct: diffPct,
|
||
Status: status,
|
||
Threshold: threshold.maxDiff,
|
||
}
|
||
}
|
||
|
||
func renderScenarioReport(report *ScenarioReport) string {
|
||
var sb strings.Builder
|
||
|
||
isSingleVersion := true
|
||
for _, c := range report.Comparisons {
|
||
if c.OldValue != 0 {
|
||
isSingleVersion = false
|
||
break
|
||
}
|
||
}
|
||
|
||
sb.WriteString("\n")
|
||
sb.WriteString("================================================================================\n")
|
||
if isSingleVersion {
|
||
sb.WriteString(" RELOADER TEST REPORT\n")
|
||
} else {
|
||
sb.WriteString(" RELOADER A/B COMPARISON REPORT\n")
|
||
}
|
||
sb.WriteString("================================================================================\n\n")
|
||
|
||
fmt.Fprintf(&sb, "Scenario: %s\n", report.Scenario)
|
||
fmt.Fprintf(&sb, "Generated: %s\n", report.Timestamp.Format("2006-01-02 15:04:05"))
|
||
fmt.Fprintf(&sb, "Status: %s\n", report.OverallStatus)
|
||
fmt.Fprintf(&sb, "Summary: %s\n", report.Summary)
|
||
|
||
if report.TestDescription != "" {
|
||
fmt.Fprintf(&sb, "Test: %s\n", report.TestDescription)
|
||
}
|
||
|
||
if report.Expected.ActionTotal > 0 {
|
||
sb.WriteString("\n--------------------------------------------------------------------------------\n")
|
||
sb.WriteString(" EXPECTED VALUES\n")
|
||
sb.WriteString("--------------------------------------------------------------------------------\n")
|
||
fmt.Fprintf(&sb, "Expected Action Total: %d\n", report.Expected.ActionTotal)
|
||
fmt.Fprintf(&sb, "Expected Reload Executed Total: %d\n", report.Expected.ReloadExecutedTotal)
|
||
if report.Expected.SkippedTotal > 0 {
|
||
fmt.Fprintf(&sb, "Expected Skipped Total: %d\n", report.Expected.SkippedTotal)
|
||
}
|
||
}
|
||
|
||
sb.WriteString("\n--------------------------------------------------------------------------------\n")
|
||
if isSingleVersion {
|
||
sb.WriteString(" METRICS\n")
|
||
} else {
|
||
sb.WriteString(" METRIC COMPARISONS\n")
|
||
}
|
||
sb.WriteString("--------------------------------------------------------------------------------\n")
|
||
|
||
if isSingleVersion {
|
||
sb.WriteString("(✓ = meets expected value within 15%)\n\n")
|
||
fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n",
|
||
"Metric", "Value", "Expected", "Met?", "Status")
|
||
fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n",
|
||
"------", "-----", "--------", "----", "------")
|
||
|
||
for _, c := range report.Comparisons {
|
||
if c.IsCounter {
|
||
if c.Expected > 0 {
|
||
fmt.Fprintf(&sb, "%-32s %12.0f %10.0f %5s %8s\n",
|
||
c.DisplayName, c.NewValue, c.Expected,
|
||
c.NewMeetsExpected, c.Status)
|
||
} else {
|
||
fmt.Fprintf(&sb, "%-32s %12.0f %10s %5s %8s\n",
|
||
c.DisplayName, c.NewValue, "-",
|
||
c.NewMeetsExpected, c.Status)
|
||
}
|
||
} else {
|
||
fmt.Fprintf(&sb, "%-32s %12.4f %10s %5s %8s\n",
|
||
c.DisplayName, c.NewValue, "-",
|
||
c.NewMeetsExpected, c.Status)
|
||
}
|
||
}
|
||
} else {
|
||
sb.WriteString("(Old✓/New✓ = meets expected value within 15%)\n\n")
|
||
|
||
fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n",
|
||
"Metric", "Old", "New", "Expected", "Old✓", "New✓", "Status")
|
||
fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n",
|
||
"------", "---", "---", "--------", "----", "----", "------")
|
||
|
||
for _, c := range report.Comparisons {
|
||
if c.IsCounter {
|
||
if c.Expected > 0 {
|
||
fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10.0f %5s %5s %8s\n",
|
||
c.DisplayName, c.OldValue, c.NewValue, c.Expected,
|
||
c.OldMeetsExpected, c.NewMeetsExpected, c.Status)
|
||
} else {
|
||
fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10s %5s %5s %8s\n",
|
||
c.DisplayName, c.OldValue, c.NewValue, "-",
|
||
c.OldMeetsExpected, c.NewMeetsExpected, c.Status)
|
||
}
|
||
} else {
|
||
fmt.Fprintf(&sb, "%-32s %12.4f %12.4f %10s %5s %5s %8s\n",
|
||
c.DisplayName, c.OldValue, c.NewValue, "-",
|
||
c.OldMeetsExpected, c.NewMeetsExpected, c.Status)
|
||
}
|
||
}
|
||
}
|
||
|
||
sb.WriteString("\n--------------------------------------------------------------------------------\n")
|
||
sb.WriteString(" PASS/FAIL CRITERIA\n")
|
||
sb.WriteString("--------------------------------------------------------------------------------\n\n")
|
||
|
||
fmt.Fprintf(&sb, "Passed (%d):\n", len(report.PassCriteria))
|
||
for _, p := range report.PassCriteria {
|
||
fmt.Fprintf(&sb, " ✓ %s\n", p)
|
||
}
|
||
|
||
if len(report.FailedCriteria) > 0 {
|
||
fmt.Fprintf(&sb, "\nFailed (%d):\n", len(report.FailedCriteria))
|
||
for _, f := range report.FailedCriteria {
|
||
fmt.Fprintf(&sb, " ✗ %s\n", f)
|
||
}
|
||
}
|
||
|
||
sb.WriteString("\n--------------------------------------------------------------------------------\n")
|
||
sb.WriteString(" THRESHOLDS USED\n")
|
||
sb.WriteString("--------------------------------------------------------------------------------\n\n")
|
||
|
||
fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n",
|
||
"Metric", "Max Diff%", "Min Abs Diff", "Direction")
|
||
fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n",
|
||
"------", "---------", "------------", "---------")
|
||
|
||
var names []string
|
||
for name := range thresholds {
|
||
names = append(names, name)
|
||
}
|
||
sort.Strings(names)
|
||
|
||
for _, name := range names {
|
||
t := thresholds[name]
|
||
var direction string
|
||
switch t.metricType {
|
||
case LowerIsBetter:
|
||
direction = "lower is better"
|
||
case HigherIsBetter:
|
||
direction = "higher is better"
|
||
case ShouldMatch:
|
||
direction = "should match"
|
||
case Informational:
|
||
direction = "info only"
|
||
}
|
||
minAbsDiff := "-"
|
||
if t.minAbsDiff > 0 {
|
||
minAbsDiff = fmt.Sprintf("%.1f", t.minAbsDiff)
|
||
}
|
||
fmt.Fprintf(&sb, "%-35s %9.1f%% %15s %18s\n",
|
||
name, t.maxDiff, minAbsDiff, direction)
|
||
}
|
||
|
||
sb.WriteString("\n================================================================================\n")
|
||
|
||
return sb.String()
|
||
}
|
||
|
||
func renderScenarioReportJSON(report *ScenarioReport) string {
|
||
data, err := json.MarshalIndent(report, "", " ")
|
||
if err != nil {
|
||
return fmt.Sprintf(`{"error": "%s"}`, err.Error())
|
||
}
|
||
return string(data)
|
||
}
|
||
|
||
func renderScenarioReportMarkdown(report *ScenarioReport) string {
|
||
var sb strings.Builder
|
||
|
||
emoji := "✅"
|
||
if report.OverallStatus != "PASS" {
|
||
emoji = "❌"
|
||
}
|
||
|
||
sb.WriteString(fmt.Sprintf("## %s %s: %s\n\n", emoji, report.Scenario, report.OverallStatus))
|
||
|
||
if report.TestDescription != "" {
|
||
sb.WriteString(fmt.Sprintf("> %s\n\n", report.TestDescription))
|
||
}
|
||
|
||
sb.WriteString("| Metric | Value | Expected | Status |\n")
|
||
sb.WriteString("|--------|------:|:--------:|:------:|\n")
|
||
|
||
keyMetrics := []string{"action_total", "reload_executed_total", "errors_total", "reconcile_total"}
|
||
for _, name := range keyMetrics {
|
||
for _, c := range report.Comparisons {
|
||
if c.Name == name {
|
||
value := fmt.Sprintf("%.0f", c.NewValue)
|
||
expected := "-"
|
||
if c.Expected > 0 {
|
||
expected = fmt.Sprintf("%.0f", c.Expected)
|
||
}
|
||
status := "✅"
|
||
if c.Status == "fail" {
|
||
status = "❌"
|
||
} else if c.Status == "info" {
|
||
status = "ℹ️"
|
||
}
|
||
sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", c.DisplayName, value, expected, status))
|
||
break
|
||
}
|
||
}
|
||
}
|
||
|
||
return sb.String()
|
||
}
|