Add Prometheus instrumentation and liveness/readiness probes

artifacts/steerer/deployment.yaml

@@ -17,7 +17,7 @@ spec:
       labels:
         app: steerer
       annotations:
-        prometheus.io.port: "8080"
+        prometheus.io/scrape: "true"
     spec:
       serviceAccountName: steerer
       containers:
@@ -25,11 +25,40 @@ spec:
         image: stefanprodan/steerer:0.0.1
         imagePullPolicy: Always
         ports:
-        - containerPort: 8080
+        - name: http
+          containerPort: 8080
         command:
         - ./steerer
         - -level=info
         - -prometheus=http://prometheus.istio-system.svc.cluster.local:9090
         - -window=10s
         - -logtostderr
-        - -v=1
+        - -stderrthreshold=ERROR
+        livenessProbe:
+          exec:
+            command:
+            - wget
+            - --quiet
+            - --tries=1
+            - --timeout=2
+            - --spider
+            - http://localhost:8080/readyz
+        readinessProbe:
+          exec:
+            command:
+            - wget
+            - --quiet
+            - --tries=1
+            - --timeout=2
+            - --spider
+            - http://localhost:8080/readyz
+        resources:
+          limits:
+            memory: "512Mi"
+            cpu: "1000m"
+          requests:
+            memory: "32Mi"
+            cpu: "10m"
+        securityContext:
+          readOnlyRootFilesystem: true
+          runAsUser: 10001

cmd/controller/main.go

@@ -5,13 +5,13 @@ import (
 	"log"
 	"time"
 
-	"github.com/golang/glog"
 	sharedclientset "github.com/knative/pkg/client/clientset/versioned"
 	"github.com/knative/pkg/signals"
 	clientset "github.com/stefanprodan/steerer/pkg/client/clientset/versioned"
 	informers "github.com/stefanprodan/steerer/pkg/client/informers/externalversions"
 	"github.com/stefanprodan/steerer/pkg/controller"
 	"github.com/stefanprodan/steerer/pkg/logging"
+	"github.com/stefanprodan/steerer/pkg/server"
 	"github.com/stefanprodan/steerer/pkg/version"
 	"go.uber.org/zap"
 	"k8s.io/client-go/kubernetes"
@@ -26,6 +26,7 @@ var (
 	metricServer  string
 	rolloutWindow time.Duration
 	logLevel      string
+	port          string
 )
 
 func init() {
@@ -34,6 +35,7 @@ func init() {
 	flag.StringVar(&metricServer, "prometheus", "http://prometheus:9090", "Prometheus URL")
 	flag.DurationVar(&rolloutWindow, "window", 10*time.Second, "wait interval between deployment rollouts")
 	flag.StringVar(&logLevel, "level", "debug", "Log level can be: debug, info, warning, error.")
+	flag.StringVar(&port, "port", "8080", "Port to listen on.")
 }
 
 func main() {
@@ -64,7 +66,7 @@ func main() {
 
 	rolloutClient, err := clientset.NewForConfig(cfg)
 	if err != nil {
-		glog.Fatalf("Error building example clientset: %s", err.Error())
+		logger.Fatalf("Error building example clientset: %s", err.Error())
 	}
 
 	rolloutInformerFactory := informers.NewSharedInformerFactory(rolloutClient, time.Second*30)
@@ -81,6 +83,9 @@ func main() {
 		zap.String("metrics provider", metricServer),
 		zap.Any("kubernetes version", ver))
 
+	// start HTTP server
+	go server.ListenAndServe(port, 3*time.Second, logger, stopCh)
+
 	c := controller.NewController(
 		kubeClient,
 		sharedClient,
@@ -98,13 +103,14 @@ func main() {
 		rolloutInformer.Informer().HasSynced,
 	} {
 		if ok := cache.WaitForCacheSync(stopCh, synced); !ok {
-			logger.Fatalf("failed to wait for cache")
+			logger.Fatalf("Failed to wait for cache sync")
 		}
 	}
 
+	// start controller
 	go func(ctrl *controller.Controller) {
-		if runErr := ctrl.Run(2, stopCh); runErr != nil {
-			logger.Fatalf("Error running controller: %v", runErr)
+		if err := ctrl.Run(2, stopCh); err != nil {
+			logger.Fatalf("Error running controller: %v", err)
 		}
 	}(c)
 

pkg/controller/controller.go

@@ -2,9 +2,8 @@ package controller
 
 import (
 	"fmt"
-	"time"
-
 	"sync"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	sharedclientset "github.com/knative/pkg/client/clientset/versioned"

pkg/server/server.go

@@ -0,0 +1,47 @@
+package server
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.uber.org/zap"
+)
+
+func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogger, stopCh <-chan struct{}) {
+	mux := http.DefaultServeMux
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("OK"))
+	})
+
+	srv := &http.Server{
+		Addr:         ":" + port,
+		Handler:      mux,
+		ReadTimeout:  5 * time.Second,
+		WriteTimeout: 1 * time.Minute,
+		IdleTimeout:  15 * time.Second,
+	}
+
+	logger.Infof("Starting HTTP server on port %s", port)
+
+	// run server in background
+	go func() {
+		if err := srv.ListenAndServe(); err != http.ErrServerClosed {
+			logger.Fatalf("HTTP server crashed %v", err)
+		}
+	}()
+
+	// wait for SIGTERM or SIGINT
+	<-stopCh
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	if err := srv.Shutdown(ctx); err != nil {
+		logger.Errorf("HTTP server graceful shutdown failed %v", err)
+	} else {
+		logger.Info("HTTP server stopped")
+	}
+}