diff --git a/cmd/kured/main.go b/cmd/kured/main.go index fb245e9..1e69877 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -80,13 +80,17 @@ func rebootRequired() bool { func rebootBlocked() bool { if prometheusURL != "" { - count, err := alerts.PrometheusCountActive(prometheusURL, alertFilter) + alertNames, err := alerts.PrometheusActiveAlerts(prometheusURL, alertFilter) if err != nil { log.Warnf("Reboot blocked: prometheus query error: %v", err) return true } + count := len(alertNames) + if count > 10 { + alertNames = append(alertNames[:10], "...") + } if count > 0 { - log.Warnf("Reboot blocked: %d active alerts", count) + log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames) return true } } diff --git a/pkg/alerts/prometheus.go b/pkg/alerts/prometheus.go index 39455df..31770e0 100644 --- a/pkg/alerts/prometheus.go +++ b/pkg/alerts/prometheus.go @@ -10,33 +10,34 @@ import ( "github.com/prometheus/common/model" ) -// Return true if there are any active (e.g. pending or firing) alerts -func PrometheusCountActive(prometheusURL string, filter *regexp.Regexp) (int, error) { +// Returns a list of names of active (e.g. pending or firing) alerts, filtered +// by the supplied regexp. +func PrometheusActiveAlerts(prometheusURL string, filter *regexp.Regexp) ([]string, error) { client, err := prometheus.New(prometheus.Config{Address: prometheusURL}) if err != nil { - return 0, err + return nil, err } queryAPI := prometheus.NewQueryAPI(client) value, err := queryAPI.Query(context.Background(), "ALERTS", time.Now()) if err != nil { - return 0, err + return nil, err } if value.Type() == model.ValVector { if vector, ok := value.(model.Vector); ok { - var count int + var activeAlerts []string for _, sample := range vector { - if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert { + if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 { if filter == nil || !filter.MatchString(string(alertName)) { - count++ + activeAlerts = append(activeAlerts, string(alertName)) } } } - return count, nil + return activeAlerts, nil } } - return 0, fmt.Errorf("Unexpected value type: %v", value) + return nil, fmt.Errorf("Unexpected value type: %v", value) }