From 6cfa4328342afc654f2fcf10d7ea5516bd9f9dd6 Mon Sep 17 00:00:00 2001
From: stefanprodan <stefan.prodan@gmail.com>
Date: Thu, 14 May 2020 10:03:44 +0300
Subject: [PATCH] Retry canary initialization on conflict

---
 pkg/controller/controller.go | 15 +++------------
 pkg/controller/finalizer.go  |  2 +-
 pkg/controller/scheduler.go  | 30 ++++++++++++++++++++++++++++++
 3 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
index 8728e3c7..25e1efb3 100644
--- a/pkg/controller/controller.go
+++ b/pkg/controller/controller.go
@@ -1,7 +1,6 @@
 package controller
 
 import (
-	"context"
 	"fmt"
 	"sync"
 	"time"
@@ -10,7 +9,6 @@ import (
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
@@ -265,16 +263,9 @@ func (c *Controller) syncHandler(key string) error {
 
 	// set status condition for new canaries
 	if cd.Status.Conditions == nil {
-		if ok, conditions := canary.MakeStatusConditions(cd, flaggerv1.CanaryPhaseInitializing); ok {
-			cdCopy := cd.DeepCopy()
-			cdCopy.Status.Conditions = conditions
-			cdCopy.Status.LastTransitionTime = metav1.Now()
-			cdCopy.Status.Phase = flaggerv1.CanaryPhaseInitializing
-			_, err := c.flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).UpdateStatus(context.TODO(), cdCopy, metav1.UpdateOptions{})
-			if err != nil {
-				c.logger.Errorf("%s status condition update error: %v", key, err)
-				return fmt.Errorf("%s status condition update error: %w", key, err)
-			}
+		if err := c.setPhaseInitializing(cd); err != nil {
+			c.logger.Errorf("%s unable to set initializing status: %v", key, err)
+			return fmt.Errorf("%s initializing error: %w", key, err)
 		}
 	}
 
diff --git a/pkg/controller/finalizer.go b/pkg/controller/finalizer.go
index 4ea2a139..34633342 100644
--- a/pkg/controller/finalizer.go
+++ b/pkg/controller/finalizer.go
@@ -44,7 +44,7 @@ func (c *Controller) finalize(old interface{}) error {
 	c.logger.Infof("%s.%s kind %s reverted", canary.Name, canary.Namespace, canary.Spec.TargetRef.Kind)
 
 	// Ensure that targetRef has met a ready state
-	c.logger.Infof("Checking is canary is ready %s.%s", canary.Name, canary.Namespace)
+	c.logger.Infof("Checking if canary is ready %s.%s", canary.Name, canary.Namespace)
 	_, err = canaryController.IsCanaryReady(canary)
 	if err != nil {
 		return fmt.Errorf("canary not ready during finalizing: %w", err)
diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go
index 5960df85..69aa9ae1 100644
--- a/pkg/controller/scheduler.go
+++ b/pkg/controller/scheduler.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
 
 	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
 	"github.com/weaveworks/flagger/pkg/canary"
@@ -738,3 +739,32 @@ func (c *Controller) rollback(canary *flaggerv1.Canary, canaryController canary.
 	c.recorder.SetStatus(canary, flaggerv1.CanaryPhaseFailed)
 	c.runPostRolloutHooks(canary, flaggerv1.CanaryPhaseFailed)
 }
+
+func (c *Controller) setPhaseInitializing(cd *flaggerv1.Canary) error {
+	phase := flaggerv1.CanaryPhaseInitializing
+	firstTry := true
+	name, ns := cd.GetName(), cd.GetNamespace()
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		if !firstTry {
+			cd, err = c.flaggerClient.FlaggerV1beta1().Canaries(ns).Get(context.TODO(), name, metav1.GetOptions{})
+			if err != nil {
+				return fmt.Errorf("canary %s.%s get query failed: %w", name, ns, err)
+			}
+		}
+
+		if ok, conditions := canary.MakeStatusConditions(cd, phase); ok {
+			cdCopy := cd.DeepCopy()
+			cdCopy.Status.Conditions = conditions
+			cdCopy.Status.LastTransitionTime = metav1.Now()
+			cdCopy.Status.Phase = phase
+			_, err = c.flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).UpdateStatus(context.TODO(), cdCopy, metav1.UpdateOptions{})
+		}
+		firstTry = false
+		return
+	})
+
+	if err != nil {
+		return fmt.Errorf("failed after retries: %w", err)
+	}
+	return nil
+}