diff --git a/pkg/goldpinger/stats.go b/pkg/goldpinger/stats.go index d5db47b..65dd9ef 100644 --- a/pkg/goldpinger/stats.go +++ b/pkg/goldpinger/stats.go @@ -48,6 +48,16 @@ var ( }, ) + goldpingerClusterHealthGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "goldpinger_cluster_health_total", + Help: "1 if all check pass, 0 otherwise", + }, + []string{ + "goldpinger_instance", + }, + ) + goldpingerResponseTimePeersHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "goldpinger_peers_response_time_s", @@ -100,6 +110,7 @@ var ( func init() { prometheus.MustRegister(goldpingerStatsCounter) prometheus.MustRegister(goldpingerNodesHealthGauge) + prometheus.MustRegister(goldpingerClusterHealthGauge) prometheus.MustRegister(goldpingerResponseTimePeersHistogram) prometheus.MustRegister(goldpingerResponseTimeKubernetesHistogram) prometheus.MustRegister(goldpingerErrorsCounter) @@ -135,6 +146,17 @@ func CountHealthyUnhealthyNodes(healthy, unhealthy float64) { ).Set(unhealthy) } +// SetClusterHealth sets the cluster health gauge to 1 (healthy) or 0 (unhealthy) +func SetClusterHealth(healthy bool) { + value := 1.0 + if healthy { + value = 0 + } + goldpingerClusterHealthGauge.WithLabelValues( + GoldpingerConfig.Hostname, + ).Set(value) +} + // counts instances of various errors func CountError(errorType string) { goldpingerErrorsCounter.WithLabelValues( diff --git a/pkg/goldpinger/updater.go b/pkg/goldpinger/updater.go index 8329f81..58c1b0f 100644 --- a/pkg/goldpinger/updater.go +++ b/pkg/goldpinger/updater.go @@ -153,6 +153,17 @@ func updateCounters() { } } CountHealthyUnhealthyNodes(counterHealthy, float64(len(checkResults.PodResults))-counterHealthy) + // check DNS, but don't block the access to checkResultsMux + go func(healthy bool) { + if healthy { + for _, response := range *checkDNS() { + if response.Error != "" { + healthy = false + } + } + } + SetClusterHealth(healthy) + }(int(counterHealthy) == len(checkResults.PodResults)) } // collectResults simply reads results from the results channel and saves them in a map