Add an overall metric goldpinger_cluster_health_total (pings + DNS check)

Signed-off-by: Mikolaj Pawlikowski <mikolaj@pawlikowski.pl>

pkg/goldpinger/stats.go

@@ -48,6 +48,16 @@ var (
 		},
 	)
 
+	goldpingerClusterHealthGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "goldpinger_cluster_health_total",
+			Help: "1 if all check pass, 0 otherwise",
+		},
+		[]string{
+			"goldpinger_instance",
+		},
+	)
+
 	goldpingerResponseTimePeersHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Name:    "goldpinger_peers_response_time_s",
@@ -100,6 +110,7 @@ var (
 func init() {
 	prometheus.MustRegister(goldpingerStatsCounter)
 	prometheus.MustRegister(goldpingerNodesHealthGauge)
+	prometheus.MustRegister(goldpingerClusterHealthGauge)
 	prometheus.MustRegister(goldpingerResponseTimePeersHistogram)
 	prometheus.MustRegister(goldpingerResponseTimeKubernetesHistogram)
 	prometheus.MustRegister(goldpingerErrorsCounter)
@@ -135,6 +146,17 @@ func CountHealthyUnhealthyNodes(healthy, unhealthy float64) {
 	).Set(unhealthy)
 }
 
+// SetClusterHealth sets the cluster health gauge to 1 (healthy) or 0 (unhealthy)
+func SetClusterHealth(healthy bool) {
+	value := 1.0
+	if healthy {
+		value = 0
+	}
+	goldpingerClusterHealthGauge.WithLabelValues(
+		GoldpingerConfig.Hostname,
+	).Set(value)
+}
+
 // counts instances of various errors
 func CountError(errorType string) {
 	goldpingerErrorsCounter.WithLabelValues(

pkg/goldpinger/updater.go

@@ -153,6 +153,17 @@ func updateCounters() {
 		}
 	}
 	CountHealthyUnhealthyNodes(counterHealthy, float64(len(checkResults.PodResults))-counterHealthy)
+	// check DNS, but don't block the access to checkResultsMux
+	go func(healthy bool) {
+		if healthy {
+			for _, response := range *checkDNS() {
+				if response.Error != "" {
+					healthy = false
+				}
+			}
+		}
+		SetClusterHealth(healthy)
+	}(int(counterHealthy) == len(checkResults.PodResults))
 }
 
 // collectResults simply reads results from the results channel and saves them in a map