From ff4aa6206129a6b32498849bef7afcefac1c5755 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 10 Jul 2019 11:31:20 +0300 Subject: [PATCH] Retry canary status update on conflict --- go.mod | 1 + pkg/canary/deployer.go | 8 --- pkg/canary/status.go | 155 +++++++++++++++++++++++++++++------------ 3 files changed, 113 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index 7100be67..59b27495 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/hashstructure v1.0.0 + github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect github.com/prometheus/common v0.3.0 // indirect diff --git a/pkg/canary/deployer.go b/pkg/canary/deployer.go index 1c5a47b5..070b72b1 100644 --- a/pkg/canary/deployer.go +++ b/pkg/canary/deployer.go @@ -120,14 +120,6 @@ func (c *Deployer) Promote(cd *flaggerv1.Canary) error { primaryCopy.GetName(), primaryCopy.Namespace, err) } - // update primary spec hash - cdClone := cd.DeepCopy() - cdClone.Status.LastPromotedSpec = cd.Status.LastAppliedSpec - _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdClone) - if err != nil { - return fmt.Errorf("updating canary status LastAppliedSpec failed: %v", err) - } - // update HPA if cd.Spec.AutoscalerRef != nil && cd.Spec.AutoscalerRef.Kind == "HorizontalPodAutoscaler" { if err := c.reconcilePrimaryHpa(cd, false); err != nil { diff --git a/pkg/canary/status.go b/pkg/canary/status.go index ef98d438..2001470c 100644 --- a/pkg/canary/status.go +++ b/pkg/canary/status.go @@ -2,8 +2,10 @@ package canary import ( "fmt" + "k8s.io/client-go/util/retry" "github.com/mitchellh/hashstructure" + ex "github.com/pkg/errors" flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -17,96 +19,163 @@ func (c *Deployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatu if errors.IsNotFound(err) { return fmt.Errorf("deployment %s.%s not found", cd.Spec.TargetRef.Name, cd.Namespace) } - return fmt.Errorf("deployment %s.%s query error %v", cd.Spec.TargetRef.Name, cd.Namespace, err) + return ex.Wrap(err, "SyncStatus deployment query error") } configs, err := c.ConfigTracker.GetConfigRefs(cd) if err != nil { - return fmt.Errorf("configs query error %v", err) + return ex.Wrap(err, "SyncStatus configs query error") } hash, err := hashstructure.Hash(dep.Spec.Template, nil) if err != nil { - return fmt.Errorf("hash error %v", err) + return ex.Wrap(err, "SyncStatus hash error") } - cdCopy := cd.DeepCopy() - cdCopy.Status.Phase = status.Phase - cdCopy.Status.CanaryWeight = status.CanaryWeight - cdCopy.Status.FailedChecks = status.FailedChecks - cdCopy.Status.Iterations = status.Iterations - cdCopy.Status.LastAppliedSpec = fmt.Sprintf("%d", hash) - cdCopy.Status.LastTransitionTime = metav1.Now() - cdCopy.Status.TrackedConfigs = configs + firstTry := true + err = retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + var selErr error + if !firstTry { + cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{}) + if selErr != nil { + return selErr + } + } + cdCopy := cd.DeepCopy() + cdCopy.Status.Phase = status.Phase + cdCopy.Status.CanaryWeight = status.CanaryWeight + cdCopy.Status.FailedChecks = status.FailedChecks + cdCopy.Status.Iterations = status.Iterations + cdCopy.Status.LastAppliedSpec = fmt.Sprintf("%d", hash) + cdCopy.Status.LastTransitionTime = metav1.Now() + cdCopy.Status.TrackedConfigs = configs - if ok, conditions := c.MakeStatusConditions(cd.Status, status.Phase); ok { - cdCopy.Status.Conditions = conditions - } + if ok, conditions := c.MakeStatusConditions(cd.Status, status.Phase); ok { + cdCopy.Status.Conditions = conditions + } - cd, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + firstTry = false + return + }) if err != nil { - return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + return ex.Wrap(err, "SyncStatus") } return nil } // SetStatusFailedChecks updates the canary failed checks counter func (c *Deployer) SetStatusFailedChecks(cd *flaggerv1.Canary, val int) error { - cdCopy := cd.DeepCopy() - cdCopy.Status.FailedChecks = val - cdCopy.Status.LastTransitionTime = metav1.Now() + firstTry := true + err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + var selErr error + if !firstTry { + cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{}) + if selErr != nil { + return selErr + } + } + cdCopy := cd.DeepCopy() + cdCopy.Status.FailedChecks = val + cdCopy.Status.LastTransitionTime = metav1.Now() - cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + firstTry = false + return + }) if err != nil { - return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + return ex.Wrap(err, "SetStatusFailedChecks") } return nil } // SetStatusWeight updates the canary status weight value func (c *Deployer) SetStatusWeight(cd *flaggerv1.Canary, val int) error { - cdCopy := cd.DeepCopy() - cdCopy.Status.CanaryWeight = val - cdCopy.Status.LastTransitionTime = metav1.Now() + firstTry := true + err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + var selErr error + if !firstTry { + cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{}) + if selErr != nil { + return selErr + } + } + cdCopy := cd.DeepCopy() + cdCopy.Status.CanaryWeight = val + cdCopy.Status.LastTransitionTime = metav1.Now() - cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + firstTry = false + return + }) if err != nil { - return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + return ex.Wrap(err, "SetStatusWeight") } return nil } // SetStatusIterations updates the canary status iterations value func (c *Deployer) SetStatusIterations(cd *flaggerv1.Canary, val int) error { - cdCopy := cd.DeepCopy() - cdCopy.Status.Iterations = val - cdCopy.Status.LastTransitionTime = metav1.Now() + firstTry := true + err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + var selErr error + if !firstTry { + cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{}) + if selErr != nil { + return selErr + } + } + + cdCopy := cd.DeepCopy() + cdCopy.Status.Iterations = val + cdCopy.Status.LastTransitionTime = metav1.Now() + + _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + firstTry = false + return + }) - cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) if err != nil { - return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + return ex.Wrap(err, "SetStatusIterations") } return nil } // SetStatusPhase updates the canary status phase func (c *Deployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) error { - cdCopy := cd.DeepCopy() - cdCopy.Status.Phase = phase - cdCopy.Status.LastTransitionTime = metav1.Now() + firstTry := true + err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + var selErr error + if !firstTry { + cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{}) + if selErr != nil { + return selErr + } + } + cdCopy := cd.DeepCopy() + cdCopy.Status.Phase = phase + cdCopy.Status.LastTransitionTime = metav1.Now() - if phase != flaggerv1.CanaryPhaseProgressing { - cdCopy.Status.CanaryWeight = 0 - cdCopy.Status.Iterations = 0 - } + if phase != flaggerv1.CanaryPhaseProgressing { + cdCopy.Status.CanaryWeight = 0 + cdCopy.Status.Iterations = 0 + } - if ok, conditions := c.MakeStatusConditions(cdCopy.Status, phase); ok { - cdCopy.Status.Conditions = conditions - } + // on promotion set primary spec hash + if phase == flaggerv1.CanaryPhaseInitialized || phase == flaggerv1.CanaryPhaseSucceeded { + cdCopy.Status.LastPromotedSpec = cd.Status.LastAppliedSpec + } - cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + if ok, conditions := c.MakeStatusConditions(cdCopy.Status, phase); ok { + cdCopy.Status.Conditions = conditions + } + + _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + firstTry = false + return + }) if err != nil { - return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + return ex.Wrap(err, "SetStatusPhase") } return nil }