Add a 5s timeout to Prometheus queries

pkg/controller/controller.go

@@ -38,7 +38,7 @@ type Controller struct {
 	workqueue     workqueue.RateLimitingInterface
 	recorder      record.EventRecorder
 	logger        *zap.SugaredLogger
-	metricServer  string
+	metricsServer string
 	rollouts      *sync.Map
 }
 
@@ -72,7 +72,7 @@ func NewController(
 		recorder:      recorder,
 		logger:        logger,
 		rollouts:      new(sync.Map),
-		metricServer:  metricServer,
+		metricsServer: metricServer,
 		rolloutWindow: rolloutWindow,
 	}
 
@@ -185,7 +185,6 @@ func (c *Controller) syncHandler(key string) error {
 
 	c.logger.Infof("Adding %s.%s to cache", rollout.Name, rollout.Namespace)
 	c.rollouts.Store(fmt.Sprintf("%s.%s", rollout.Name, rollout.Namespace), rollout)
-	//c.recorder.Event(rollout, corev1.EventTypeNormal, "Synced", "Rollout synced successfully with internal cache")
 
 	return nil
 }

pkg/controller/deployment.go

@@ -183,7 +183,7 @@ func (c *Controller) getDeployment(name string, namespace string) (*appsv1.Deplo
 func (c *Controller) checkDeploymentSuccessRate(r *rolloutv1.Rollout) bool {
 	val, err := c.getDeploymentMetric(r.Spec.Canary.Name, r.Namespace, r.Spec.Metric.Name, r.Spec.Metric.Interval)
 	if err != nil {
-		c.recordEventErrorf(r, "Metric query error: %v", err)
+		c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.metricsServer, err)
 		return false
 	}
 

pkg/controller/observer.go

@@ -1,12 +1,14 @@
 package controller
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strconv"
+	"time"
 )
 
 type VectorQueryResponse struct {
@@ -22,7 +24,7 @@ type VectorQueryResponse struct {
 }
 
 func (c *Controller) queryMetric(query string) (*VectorQueryResponse, error) {
-	promURL, err := url.Parse(c.metricServer)
+	promURL, err := url.Parse(c.metricsServer)
 	if err != nil {
 		return nil, err
 	}
@@ -33,11 +35,21 @@ func (c *Controller) queryMetric(query string) (*VectorQueryResponse, error) {
 	}
 
 	u = promURL.ResolveReference(u)
-	r, err := http.Get(u.String())
+
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithTimeout(req.Context(), 5*time.Second)
+	defer cancel()
+
+	r, err := http.DefaultClient.Do(req.WithContext(ctx))
 	if err != nil {
 		return nil, err
 	}
 	defer r.Body.Close()
+
 	b, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		return nil, fmt.Errorf("error reading body: %s", err.Error())
@@ -46,8 +58,8 @@ func (c *Controller) queryMetric(query string) (*VectorQueryResponse, error) {
 	if 400 <= r.StatusCode {
 		return nil, fmt.Errorf("error response: %s", string(b))
 	}
-	var values VectorQueryResponse
 
+	var values VectorQueryResponse
 	err = json.Unmarshal(b, &values)
 	if err != nil {
 		return nil, fmt.Errorf("error unmarshaling result: %s, '%s'", err.Error(), string(b))