mirror of
https://github.com/kubereboot/kured.git
synced 2026-05-17 22:06:53 +00:00
122 lines
3.8 KiB
Go
122 lines
3.8 KiB
Go
package blockers
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"regexp"
|
|
"sort"
|
|
"time"
|
|
|
|
papi "github.com/prometheus/client_golang/api"
|
|
v1 "github.com/prometheus/client_golang/api/prometheus/v1"
|
|
"github.com/prometheus/common/model"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// Compile-time checks to ensure the type implements the interface
|
|
var (
|
|
_ RebootBlocker = (*PrometheusBlockingChecker)(nil)
|
|
)
|
|
|
|
// PrometheusBlockingChecker contains info for connecting
|
|
// to prometheus, and can give info about whether a reboot should be blocked
|
|
type PrometheusBlockingChecker struct {
|
|
promConfig papi.Config
|
|
// regexp used to get alerts
|
|
filter *regexp.Regexp
|
|
// bool to indicate if only firing alerts should be considered
|
|
firingOnly bool
|
|
// bool to indicate that we're only blocking on alerts which match the filter
|
|
filterMatchOnly bool
|
|
// storing the promClient
|
|
promClient papi.Client
|
|
}
|
|
|
|
// NewPrometheusBlockingChecker creates a new PrometheusBlockingChecker using the given
|
|
// Prometheus API config, alert filter, and filtering options.
|
|
func NewPrometheusBlockingChecker(config papi.Config, alertFilter *regexp.Regexp, firingOnly bool, filterMatchOnly bool) PrometheusBlockingChecker {
|
|
promClient, _ := papi.NewClient(config)
|
|
|
|
return PrometheusBlockingChecker{
|
|
promConfig: config,
|
|
filter: alertFilter,
|
|
firingOnly: firingOnly,
|
|
filterMatchOnly: filterMatchOnly,
|
|
promClient: promClient,
|
|
}
|
|
}
|
|
|
|
// IsBlocked for the prometheus will check if there are active alerts matching
|
|
// the arguments given into the PrometheusBlockingChecker which would actively
|
|
// block the reboot.
|
|
// As of today, no blocker information is shared as a return of the method,
|
|
// and the information is simply logged.
|
|
func (pb PrometheusBlockingChecker) IsBlocked() bool {
|
|
alertNames, err := pb.ActiveAlerts()
|
|
if err != nil {
|
|
log.Warnf("Reboot blocked: prometheus query error: %v", err)
|
|
return true
|
|
}
|
|
count := len(alertNames)
|
|
if count > 10 {
|
|
alertNames = append(alertNames[:10], "...")
|
|
}
|
|
if count > 0 {
|
|
log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames)
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// MetricLabel is used to give a fancier name
|
|
// than the type to the label for rebootBlockedCounter
|
|
func (pb PrometheusBlockingChecker) MetricLabel() string {
|
|
return "prometheus"
|
|
}
|
|
|
|
// ActiveAlerts is a method of type promClient, it returns a list of names of active alerts
|
|
// (e.g. pending or firing), filtered by the supplied regexp or by the includeLabels query.
|
|
// filter by regexp means when the regexp finds the alert-name; the alert is excluded from the
|
|
// block-list and will NOT block rebooting. query by includeLabel means,
|
|
// if the query finds an alert, it will include it to the block-list, and it WILL block rebooting.
|
|
func (pb PrometheusBlockingChecker) ActiveAlerts() ([]string, error) {
|
|
api := v1.NewAPI(pb.promClient)
|
|
|
|
// get all alerts from prometheus
|
|
value, _, err := api.Query(context.Background(), "ALERTS", time.Now())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if value.Type() == model.ValVector {
|
|
if vector, ok := value.(model.Vector); ok {
|
|
activeAlertSet := make(map[string]bool)
|
|
for _, sample := range vector {
|
|
if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 {
|
|
if matchesRegex(pb.filter, string(alertName), pb.filterMatchOnly) && (!pb.firingOnly || sample.Metric["alertstate"] == "firing") {
|
|
activeAlertSet[string(alertName)] = true
|
|
}
|
|
}
|
|
}
|
|
|
|
var activeAlerts []string
|
|
for activeAlert := range activeAlertSet {
|
|
activeAlerts = append(activeAlerts, activeAlert)
|
|
}
|
|
sort.Strings(activeAlerts)
|
|
|
|
return activeAlerts, nil
|
|
}
|
|
}
|
|
|
|
return nil, fmt.Errorf("unexpected value type %v", value)
|
|
}
|
|
|
|
func matchesRegex(filter *regexp.Regexp, alertName string, filterMatchOnly bool) bool {
|
|
if filter == nil {
|
|
return true
|
|
}
|
|
|
|
return filter.MatchString(alertName) == filterMatchOnly
|
|
}
|