Retry canary status update on conflict

2026-04-15 06:57:34 +00:00 · 2019-07-10 11:31:20 +03:00
parent 9b6cfdeef7
commit ff4aa62061
3 changed files with 113 additions and 51 deletions
@@ -31,6 +31,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
 	github.com/mitchellh/hashstructure v1.0.0
+	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect
 	github.com/prometheus/common v0.3.0 // indirect
@@ -120,14 +120,6 @@ func (c *Deployer) Promote(cd *flaggerv1.Canary) error {
 			primaryCopy.GetName(), primaryCopy.Namespace, err)
 	}

-	// update primary spec hash
-	cdClone := cd.DeepCopy()
-	cdClone.Status.LastPromotedSpec = cd.Status.LastAppliedSpec
-	_, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdClone)
-	if err != nil {
-		return fmt.Errorf("updating canary status LastAppliedSpec failed: %v", err)
-	}
-
 	// update HPA
 	if cd.Spec.AutoscalerRef != nil && cd.Spec.AutoscalerRef.Kind == "HorizontalPodAutoscaler" {
 		if err := c.reconcilePrimaryHpa(cd, false); err != nil {
@@ -2,8 +2,10 @@ package canary

 import (
 	"fmt"
+	"k8s.io/client-go/util/retry"

 	"github.com/mitchellh/hashstructure"
+	ex "github.com/pkg/errors"
 	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -17,96 +19,163 @@ func (c *Deployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatu
 		if errors.IsNotFound(err) {
 			return fmt.Errorf("deployment %s.%s not found", cd.Spec.TargetRef.Name, cd.Namespace)
 		}
-		return fmt.Errorf("deployment %s.%s query error %v", cd.Spec.TargetRef.Name, cd.Namespace, err)
+		return ex.Wrap(err, "SyncStatus deployment query error")
 	}

 	configs, err := c.ConfigTracker.GetConfigRefs(cd)
 	if err != nil {
-		return fmt.Errorf("configs query error %v", err)
+		return ex.Wrap(err, "SyncStatus configs query error")
 	}

 	hash, err := hashstructure.Hash(dep.Spec.Template, nil)
 	if err != nil {
-		return fmt.Errorf("hash error %v", err)
+		return ex.Wrap(err, "SyncStatus hash error")
 	}

-	cdCopy := cd.DeepCopy()
-	cdCopy.Status.Phase = status.Phase
-	cdCopy.Status.CanaryWeight = status.CanaryWeight
-	cdCopy.Status.FailedChecks = status.FailedChecks
-	cdCopy.Status.Iterations = status.Iterations
-	cdCopy.Status.LastAppliedSpec = fmt.Sprintf("%d", hash)
-	cdCopy.Status.LastTransitionTime = metav1.Now()
-	cdCopy.Status.TrackedConfigs = configs
+	firstTry := true
+	err = retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		var selErr error
+		if !firstTry {
+			cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+		cdCopy := cd.DeepCopy()
+		cdCopy.Status.Phase = status.Phase
+		cdCopy.Status.CanaryWeight = status.CanaryWeight
+		cdCopy.Status.FailedChecks = status.FailedChecks
+		cdCopy.Status.Iterations = status.Iterations
+		cdCopy.Status.LastAppliedSpec = fmt.Sprintf("%d", hash)
+		cdCopy.Status.LastTransitionTime = metav1.Now()
+		cdCopy.Status.TrackedConfigs = configs

-	if ok, conditions := c.MakeStatusConditions(cd.Status, status.Phase); ok {
-		cdCopy.Status.Conditions = conditions
-	}
+		if ok, conditions := c.MakeStatusConditions(cd.Status, status.Phase); ok {
+			cdCopy.Status.Conditions = conditions
+		}

-	cd, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		_, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		firstTry = false
+		return
+	})
 	if err != nil {
-		return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
+		return ex.Wrap(err, "SyncStatus")
 	}
 	return nil
 }

 // SetStatusFailedChecks updates the canary failed checks counter
 func (c *Deployer) SetStatusFailedChecks(cd *flaggerv1.Canary, val int) error {
-	cdCopy := cd.DeepCopy()
-	cdCopy.Status.FailedChecks = val
-	cdCopy.Status.LastTransitionTime = metav1.Now()
+	firstTry := true
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		var selErr error
+		if !firstTry {
+			cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+		cdCopy := cd.DeepCopy()
+		cdCopy.Status.FailedChecks = val
+		cdCopy.Status.LastTransitionTime = metav1.Now()

-	cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		_, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		firstTry = false
+		return
+	})
 	if err != nil {
-		return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
+		return ex.Wrap(err, "SetStatusFailedChecks")
 	}
 	return nil
 }

 // SetStatusWeight updates the canary status weight value
 func (c *Deployer) SetStatusWeight(cd *flaggerv1.Canary, val int) error {
-	cdCopy := cd.DeepCopy()
-	cdCopy.Status.CanaryWeight = val
-	cdCopy.Status.LastTransitionTime = metav1.Now()
+	firstTry := true
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		var selErr error
+		if !firstTry {
+			cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+		cdCopy := cd.DeepCopy()
+		cdCopy.Status.CanaryWeight = val
+		cdCopy.Status.LastTransitionTime = metav1.Now()

-	cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		_, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		firstTry = false
+		return
+	})
 	if err != nil {
-		return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
+		return ex.Wrap(err, "SetStatusWeight")
 	}
 	return nil
 }

 // SetStatusIterations updates the canary status iterations value
 func (c *Deployer) SetStatusIterations(cd *flaggerv1.Canary, val int) error {
-	cdCopy := cd.DeepCopy()
-	cdCopy.Status.Iterations = val
-	cdCopy.Status.LastTransitionTime = metav1.Now()
+	firstTry := true
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		var selErr error
+		if !firstTry {
+			cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+
+		cdCopy := cd.DeepCopy()
+		cdCopy.Status.Iterations = val
+		cdCopy.Status.LastTransitionTime = metav1.Now()
+
+		_, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		firstTry = false
+		return
+	})

-	cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
 	if err != nil {
-		return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
+		return ex.Wrap(err, "SetStatusIterations")
 	}
 	return nil
 }

 // SetStatusPhase updates the canary status phase
 func (c *Deployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) error {
-	cdCopy := cd.DeepCopy()
-	cdCopy.Status.Phase = phase
-	cdCopy.Status.LastTransitionTime = metav1.Now()
+	firstTry := true
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		var selErr error
+		if !firstTry {
+			cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+		cdCopy := cd.DeepCopy()
+		cdCopy.Status.Phase = phase
+		cdCopy.Status.LastTransitionTime = metav1.Now()

-	if phase != flaggerv1.CanaryPhaseProgressing {
-		cdCopy.Status.CanaryWeight = 0
-		cdCopy.Status.Iterations = 0
-	}
+		if phase != flaggerv1.CanaryPhaseProgressing {
+			cdCopy.Status.CanaryWeight = 0
+			cdCopy.Status.Iterations = 0
+		}

-	if ok, conditions := c.MakeStatusConditions(cdCopy.Status, phase); ok {
-		cdCopy.Status.Conditions = conditions
-	}
+		// on promotion set primary spec hash
+		if phase == flaggerv1.CanaryPhaseInitialized || phase == flaggerv1.CanaryPhaseSucceeded {
+			cdCopy.Status.LastPromotedSpec = cd.Status.LastAppliedSpec
+		}

-	cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		if ok, conditions := c.MakeStatusConditions(cdCopy.Status, phase); ok {
+			cdCopy.Status.Conditions = conditions
+		}
+
+		_, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		firstTry = false
+		return
+	})
 	if err != nil {
-		return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
+		return ex.Wrap(err, "SetStatusPhase")
 	}
 	return nil
 }