feat: add support for webhook retries

Add a new field `.spec.webhooks[].retries` to specify the number of
retries when calling a webhook.

Signed-off-by: Joseph Kwasniewski <kwasniewski@gmail.com>

pkg/apis/flagger/v1beta1/canary.go

@@ -394,6 +394,10 @@ type CanaryWebhook struct {
 	// Metadata (key-value pairs) for this webhook
 	// +optional
 	Metadata *map[string]string `json:"metadata,omitempty"`
+
+	// Number of retries for this webhook
+	// +optional
+	Retries int `json:"retries,omitempty"`
 }
 
 // CanaryWebhookPayload holds the deployment info and metadata sent to webhooks

pkg/controller/webhook.go

@@ -18,21 +18,21 @@ package controller
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
-	"net/http"
 	"net/url"
 	"strconv"
 	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
+
 	flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
 	"github.com/fluxcd/flagger/pkg/canary"
 )
 
-func callWebhook(webhook string, payload interface{}, timeout string) error {
+func callWebhook(webhook string, payload interface{}, timeout string, retries int) error {
 	payloadBin, err := json.Marshal(payload)
 	if err != nil {
 		return err
@@ -43,7 +43,11 @@ func callWebhook(webhook string, payload interface{}, timeout string) error {
 		return err
 	}
 
-	req, err := http.NewRequest("POST", hook.String(), bytes.NewBuffer(payloadBin))
+	httpClient := retryablehttp.NewClient()
+	httpClient.RetryMax = retries
+	httpClient.Logger = nil
+
+	req, err := retryablehttp.NewRequest("POST", hook.String(), bytes.NewBuffer(payloadBin))
 	if err != nil {
 		return err
 	}
@@ -53,16 +57,14 @@ func callWebhook(webhook string, payload interface{}, timeout string) error {
 	if timeout == "" {
 		timeout = "10s"
 	}
-
 	t, err := time.ParseDuration(timeout)
 	if err != nil {
 		return err
 	}
 
-	ctx, cancel := context.WithTimeout(req.Context(), t)
-	defer cancel()
+	httpClient.HTTPClient.Timeout = t
 
-	r, err := http.DefaultClient.Do(req.WithContext(ctx))
+	r, err := httpClient.Do(req)
 	if err != nil {
 		return err
 	}
@@ -98,7 +100,7 @@ func CallWebhook(canary flaggerv1.Canary, phase flaggerv1.CanaryPhase, w flagger
 		w.Timeout = "10s"
 	}
 
-	return callWebhook(w.URL, payload, w.Timeout)
+	return callWebhook(w.URL, payload, w.Timeout, w.Retries)
 }
 
 func CallEventWebhook(r *flaggerv1.Canary, w flaggerv1.CanaryWebhook, message, eventtype string) error {
@@ -124,7 +126,7 @@ func CallEventWebhook(r *flaggerv1.Canary, w flaggerv1.CanaryWebhook, message, e
 			payload.Metadata[key] = value
 		}
 	}
-	return callWebhook(w.URL, payload, "5s")
+	return callWebhook(w.URL, payload, "5s", w.Retries)
 }
 
 func canaryChecksum(c flaggerv1.Canary) string {

pkg/controller/webhook_test.go

@@ -263,3 +263,29 @@ func TestCanaryChecksum(t *testing.T) {
 	require.NotEqual(t, canary3sum, canary1sum)
 	require.NotEqual(t, canary4sum, canary1sum)
 }
+
+func TestCallWebhook_Retries(t *testing.T) {
+	retries := 1
+	failures := 0
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if failures <= retries-1 {
+			w.WriteHeader(http.StatusInternalServerError)
+			failures++
+		} else {
+			w.WriteHeader(http.StatusAccepted)
+		}
+	}))
+	defer ts.Close()
+	hook := flaggerv1.CanaryWebhook{
+		Name:    "validation",
+		URL:     ts.URL,
+		Retries: retries,
+	}
+
+	err := CallWebhook(
+		flaggerv1.Canary{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "podinfo", Namespace: corev1.NamespaceDefault}},
+		flaggerv1.CanaryPhaseProgressing, hook)
+	require.NoError(t, err)
+}