Improved active alert detection and logging

Zero-valued alerts are now ignored, and the names of the first ten
blocking alerts are logged for diagnostic purposes.

cmd/kured/main.go

@@ -80,13 +80,17 @@ func rebootRequired() bool {
 
 func rebootBlocked() bool {
 	if prometheusURL != "" {
-		count, err := alerts.PrometheusCountActive(prometheusURL, alertFilter)
+		alertNames, err := alerts.PrometheusActiveAlerts(prometheusURL, alertFilter)
 		if err != nil {
 			log.Warnf("Reboot blocked: prometheus query error: %v", err)
 			return true
 		}
+		count := len(alertNames)
+		if count > 10 {
+			alertNames = append(alertNames[:10], "...")
+		}
 		if count > 0 {
-			log.Warnf("Reboot blocked: %d active alerts", count)
+			log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames)
 			return true
 		}
 	}

pkg/alerts/prometheus.go

@@ -10,33 +10,34 @@ import (
 	"github.com/prometheus/common/model"
 )
 
-// Return true if there are any active (e.g. pending or firing) alerts
-func PrometheusCountActive(prometheusURL string, filter *regexp.Regexp) (int, error) {
+// Returns a list of names of active (e.g. pending or firing) alerts, filtered
+// by the supplied regexp.
+func PrometheusActiveAlerts(prometheusURL string, filter *regexp.Regexp) ([]string, error) {
 	client, err := prometheus.New(prometheus.Config{Address: prometheusURL})
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
 
 	queryAPI := prometheus.NewQueryAPI(client)
 
 	value, err := queryAPI.Query(context.Background(), "ALERTS", time.Now())
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
 
 	if value.Type() == model.ValVector {
 		if vector, ok := value.(model.Vector); ok {
-			var count int
+			var activeAlerts []string
 			for _, sample := range vector {
-				if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert {
+				if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 {
 					if filter == nil || !filter.MatchString(string(alertName)) {
-						count++
+						activeAlerts = append(activeAlerts, string(alertName))
 					}
 				}
 			}
-			return count, nil
+			return activeAlerts, nil
 		}
 	}
 
-	return 0, fmt.Errorf("Unexpected value type: %v", value)
+	return nil, fmt.Errorf("Unexpected value type: %v", value)
 }