From 3e84799644ced2b2e074de29fc7211c085b55a58 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 9 Jul 2019 08:52:31 +0300 Subject: [PATCH 01/10] Detect changes in pod template metadata Use the pod template spec hash to track changes (breaking) --- go.mod | 1 + pkg/canary/deployer.go | 21 ++++++--------------- pkg/canary/status.go | 15 +++++++-------- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 4755ac4f..7100be67 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/mattn/go-isatty v0.0.7 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect + github.com/mitchellh/hashstructure v1.0.0 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect github.com/prometheus/common v0.3.0 // indirect diff --git a/pkg/canary/deployer.go b/pkg/canary/deployer.go index 98485d2a..aa83eab3 100644 --- a/pkg/canary/deployer.go +++ b/pkg/canary/deployer.go @@ -2,20 +2,18 @@ package canary import ( "crypto/rand" - "encoding/base64" - "encoding/json" "fmt" + "io" + "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" + "github.com/mitchellh/hashstructure" flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned" "go.uber.org/zap" - "io" appsv1 "k8s.io/api/apps/v1" hpav1 "k8s.io/api/autoscaling/v2beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" @@ -146,19 +144,12 @@ func (c *Deployer) HasDeploymentChanged(cd *flaggerv1.Canary) (bool, error) { return true, nil } - newSpec := &canary.Spec.Template.Spec - oldSpecJson, err := base64.StdEncoding.DecodeString(cd.Status.LastAppliedSpec) + newHash, err := hashstructure.Hash(canary.Spec.Template, nil) if err != nil { - return false, fmt.Errorf("%s.%s decode error %v", cd.Name, cd.Namespace, err) - } - oldSpec := &corev1.PodSpec{} - err = json.Unmarshal(oldSpecJson, oldSpec) - if err != nil { - return false, fmt.Errorf("%s.%s unmarshal error %v", cd.Name, cd.Namespace, err) + return false, fmt.Errorf("hash error %v", err) } - if diff := cmp.Diff(*newSpec, *oldSpec, cmpopts.IgnoreUnexported(resource.Quantity{})); diff != "" { - //fmt.Println(diff) + if cd.Status.LastAppliedSpec != fmt.Sprintf("%d", newHash) { return true, nil } diff --git a/pkg/canary/status.go b/pkg/canary/status.go index e613e570..7c2cb317 100644 --- a/pkg/canary/status.go +++ b/pkg/canary/status.go @@ -1,10 +1,9 @@ package canary import ( - "encoding/base64" - "encoding/json" "fmt" + "github.com/mitchellh/hashstructure" flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -20,22 +19,22 @@ func (c *Deployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatu return fmt.Errorf("deployment %s.%s query error %v", cd.Spec.TargetRef.Name, cd.Namespace, err) } - specJson, err := json.Marshal(dep.Spec.Template.Spec) - if err != nil { - return fmt.Errorf("deployment %s.%s marshal error %v", cd.Spec.TargetRef.Name, cd.Namespace, err) - } - configs, err := c.ConfigTracker.GetConfigRefs(cd) if err != nil { return fmt.Errorf("configs query error %v", err) } + hash, err := hashstructure.Hash(dep.Spec.Template, nil) + if err != nil { + return fmt.Errorf("hash error %v", err) + } + cdCopy := cd.DeepCopy() cdCopy.Status.Phase = status.Phase cdCopy.Status.CanaryWeight = status.CanaryWeight cdCopy.Status.FailedChecks = status.FailedChecks cdCopy.Status.Iterations = status.Iterations - cdCopy.Status.LastAppliedSpec = base64.StdEncoding.EncodeToString(specJson) + cdCopy.Status.LastAppliedSpec = fmt.Sprintf("%d", hash) cdCopy.Status.LastTransitionTime = metav1.Now() cdCopy.Status.TrackedConfigs = configs From 438f952128690df9ea68f46a02218d308239a115 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 9 Jul 2019 15:22:56 +0300 Subject: [PATCH 02/10] Implement status conditions Add Promoted status condition with the following reasons: Initialized, Progressing, Succeeded, Failed Usage: `kubectl wait canary/app --for=condition=promoted` Fix: #184 --- pkg/apis/flagger/v1alpha3/status.go | 68 +++++++++++++++++++ pkg/apis/flagger/v1alpha3/types.go | 31 --------- .../flagger/v1alpha3/zz_generated.deepcopy.go | 25 +++++++ pkg/canary/deployer_test.go | 8 +-- pkg/canary/status.go | 66 +++++++++++++++++- pkg/controller/scheduler.go | 44 ++++++------ pkg/controller/scheduler_test.go | 18 ++--- pkg/controller/webhook_test.go | 4 +- pkg/metrics/recorder.go | 4 +- 9 files changed, 197 insertions(+), 71 deletions(-) create mode 100644 pkg/apis/flagger/v1alpha3/status.go diff --git a/pkg/apis/flagger/v1alpha3/status.go b/pkg/apis/flagger/v1alpha3/status.go new file mode 100644 index 00000000..d28a860c --- /dev/null +++ b/pkg/apis/flagger/v1alpha3/status.go @@ -0,0 +1,68 @@ +package v1alpha3 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CanaryConditionType is the type of a CanaryCondition +type CanaryConditionType string + +const ( + // PromotedType refers to the result of the last canary analysis + PromotedType CanaryConditionType = "Promoted" +) + +// CanaryCondition is a status condition for a Canary +type CanaryCondition struct { + // Type of this condition + Type CanaryConditionType `json:"type"` + + // Status of this condition + Status corev1.ConditionStatus `json:"status"` + + // LastUpdateTime of this condition + LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` + + // LastTransitionTime of this condition + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + + // Reason for the current status of this condition + Reason string `json:"reason,omitempty"` + + // Message associated with this condition + Message string `json:"message,omitempty"` +} + +// CanaryPhase is a label for the condition of a canary at the current time +type CanaryPhase string + +const ( + // CanaryPhaseInitialized means the primary deployment, hpa and ClusterIP services + // have been created along with the service mesh or ingress objects + CanaryPhaseInitialized CanaryPhase = "Initialized" + // CanaryPhaseProgressing means the canary analysis is underway + CanaryPhaseProgressing CanaryPhase = "Progressing" + // CanaryPhaseSucceeded means the canary analysis has been successful + // and the canary deployment has been promoted + CanaryPhaseSucceeded CanaryPhase = "Succeeded" + // CanaryPhaseFailed means the canary analysis failed + // and the canary deployment has been scaled to zero + CanaryPhaseFailed CanaryPhase = "Failed" +) + +// CanaryStatus is used for state persistence (read-only) +type CanaryStatus struct { + Phase CanaryPhase `json:"phase"` + FailedChecks int `json:"failedChecks"` + CanaryWeight int `json:"canaryWeight"` + Iterations int `json:"iterations"` + // +optional + TrackedConfigs *map[string]string `json:"trackedConfigs,omitempty"` + // +optional + LastAppliedSpec string `json:"lastAppliedSpec,omitempty"` + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + // +optional + Conditions []CanaryCondition `json:"conditions,omitempty"` +} diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index d127a23b..1f995e02 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -85,37 +85,6 @@ type CanaryList struct { Items []Canary `json:"items"` } -// CanaryPhase is a label for the condition of a canary at the current time -type CanaryPhase string - -const ( - // CanaryInitialized means the primary deployment, hpa and ClusterIP services - // have been created along with the Istio virtual service - CanaryInitialized CanaryPhase = "Initialized" - // CanaryProgressing means the canary analysis is underway - CanaryProgressing CanaryPhase = "Progressing" - // CanarySucceeded means the canary analysis has been successful - // and the canary deployment has been promoted - CanarySucceeded CanaryPhase = "Succeeded" - // CanaryFailed means the canary analysis failed - // and the canary deployment has been scaled to zero - CanaryFailed CanaryPhase = "Failed" -) - -// CanaryStatus is used for state persistence (read-only) -type CanaryStatus struct { - Phase CanaryPhase `json:"phase"` - FailedChecks int `json:"failedChecks"` - CanaryWeight int `json:"canaryWeight"` - Iterations int `json:"iterations"` - // +optional - TrackedConfigs *map[string]string `json:"trackedConfigs,omitempty"` - // +optional - LastAppliedSpec string `json:"lastAppliedSpec,omitempty"` - // +optional - LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` -} - // CanaryService is used to create ClusterIP services // and Istio Virtual Service type CanaryService struct { diff --git a/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go index f1210e8c..79e7d71d 100644 --- a/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go @@ -89,6 +89,24 @@ func (in *CanaryAnalysis) DeepCopy() *CanaryAnalysis { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CanaryCondition) DeepCopyInto(out *CanaryCondition) { + *out = *in + in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryCondition. +func (in *CanaryCondition) DeepCopy() *CanaryCondition { + if in == nil { + return nil + } + out := new(CanaryCondition) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CanaryList) DeepCopyInto(out *CanaryList) { *out = *in @@ -250,6 +268,13 @@ func (in *CanaryStatus) DeepCopyInto(out *CanaryStatus) { } } in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]CanaryCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } diff --git a/pkg/canary/deployer_test.go b/pkg/canary/deployer_test.go index 7a0d1c9b..ec545aa9 100644 --- a/pkg/canary/deployer_test.go +++ b/pkg/canary/deployer_test.go @@ -229,7 +229,7 @@ func TestCanaryDeployer_SetState(t *testing.T) { t.Fatal(err.Error()) } - err = mocks.deployer.SetStatusPhase(mocks.canary, v1alpha3.CanaryProgressing) + err = mocks.deployer.SetStatusPhase(mocks.canary, v1alpha3.CanaryPhaseProgressing) if err != nil { t.Fatal(err.Error()) } @@ -239,8 +239,8 @@ func TestCanaryDeployer_SetState(t *testing.T) { t.Fatal(err.Error()) } - if res.Status.Phase != v1alpha3.CanaryProgressing { - t.Errorf("Got %v wanted %v", res.Status.Phase, v1alpha3.CanaryProgressing) + if res.Status.Phase != v1alpha3.CanaryPhaseProgressing { + t.Errorf("Got %v wanted %v", res.Status.Phase, v1alpha3.CanaryPhaseProgressing) } } @@ -252,7 +252,7 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) { } status := v1alpha3.CanaryStatus{ - Phase: v1alpha3.CanaryProgressing, + Phase: v1alpha3.CanaryPhaseProgressing, FailedChecks: 2, } err = mocks.deployer.SyncStatus(mocks.canary, status) diff --git a/pkg/canary/status.go b/pkg/canary/status.go index 7c2cb317..163b0bee 100644 --- a/pkg/canary/status.go +++ b/pkg/canary/status.go @@ -5,6 +5,7 @@ import ( "github.com/mitchellh/hashstructure" flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -38,6 +39,10 @@ func (c *Deployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatu cdCopy.Status.LastTransitionTime = metav1.Now() cdCopy.Status.TrackedConfigs = configs + if ok, conditions := c.makeStatusConditions(cd.Status, status.Phase); ok { + cdCopy.Status.Conditions = conditions + } + cd, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) if err != nil { return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) @@ -90,14 +95,73 @@ func (c *Deployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPh cdCopy.Status.Phase = phase cdCopy.Status.LastTransitionTime = metav1.Now() - if phase != flaggerv1.CanaryProgressing { + if phase != flaggerv1.CanaryPhaseProgressing { cdCopy.Status.CanaryWeight = 0 cdCopy.Status.Iterations = 0 } + if ok, conditions := c.makeStatusConditions(cdCopy.Status, phase); ok { + cdCopy.Status.Conditions = conditions + } + cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) if err != nil { return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) } return nil } + +// GetStatusCondition returns a condition based on type +func (c *Deployer) getStatusCondition(status flaggerv1.CanaryStatus, conditionType flaggerv1.CanaryConditionType) *flaggerv1.CanaryCondition { + for i := range status.Conditions { + c := status.Conditions[i] + if c.Type == conditionType { + return &c + } + } + return nil +} + +// MakeStatusCondition updates the canary status conditions based on canary phase +func (c *Deployer) makeStatusConditions(canaryStatus flaggerv1.CanaryStatus, + phase flaggerv1.CanaryPhase) (bool, []flaggerv1.CanaryCondition) { + currentCondition := c.getStatusCondition(canaryStatus, flaggerv1.PromotedType) + + message := "New deployment detected, starting initialization." + status := corev1.ConditionUnknown + switch phase { + case flaggerv1.CanaryPhaseProgressing: + status = corev1.ConditionUnknown + message = "New revision detected, starting canary analysis." + case flaggerv1.CanaryPhaseInitialized: + status = corev1.ConditionTrue + message = "New deployment detected, initialization completed." + case flaggerv1.CanaryPhaseSucceeded: + status = corev1.ConditionTrue + message = "Canary analysis completed successfully, promotion finished." + case flaggerv1.CanaryPhaseFailed: + status = corev1.ConditionFalse + message = "Canary analysis failed, deployment scaled to zero." + } + + newCondition := &flaggerv1.CanaryCondition{ + Type: flaggerv1.PromotedType, + Status: status, + LastUpdateTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Message: message, + Reason: string(phase), + } + + if currentCondition != nil && + currentCondition.Status == newCondition.Status && + currentCondition.Reason == newCondition.Reason { + return false, nil + } + + if currentCondition != nil && currentCondition.Status == newCondition.Status { + newCondition.LastTransitionTime = currentCondition.LastTransitionTime + } + + return true, []flaggerv1.CanaryCondition{*newCondition} +} diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 5ed080ad..10cbca22 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -178,7 +178,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh // reset status status := flaggerv1.CanaryStatus{ - Phase: flaggerv1.CanaryProgressing, + Phase: flaggerv1.CanaryPhaseProgressing, CanaryWeight: 0, FailedChecks: 0, Iterations: 0, @@ -210,7 +210,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh } // check if the number of failed checks reached the threshold - if cd.Status.Phase == flaggerv1.CanaryProgressing && + if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing && (!retriable || cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold) { if cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold { @@ -246,13 +246,13 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh } // mark canary as failed - if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed, CanaryWeight: 0}); err != nil { + if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseFailed, CanaryWeight: 0}); err != nil { c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err) return } - c.recorder.SetStatus(cd, flaggerv1.CanaryFailed) - c.runPostRolloutHooks(cd, flaggerv1.CanaryFailed) + c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseFailed) + c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseFailed) return } @@ -331,12 +331,12 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh } // update status phase - if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil { + if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil { c.recordEventWarningf(cd, "%v", err) return } - c.recorder.SetStatus(cd, flaggerv1.CanarySucceeded) - c.runPostRolloutHooks(cd, flaggerv1.CanarySucceeded) + c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded) + c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded) c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.", false, false) return @@ -398,12 +398,12 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh } // update status phase - if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil { + if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil { c.recordEventWarningf(cd, "%v", err) return } - c.recorder.SetStatus(cd, flaggerv1.CanarySucceeded) - c.runPostRolloutHooks(cd, flaggerv1.CanarySucceeded) + c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded) + c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded) c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.", false, false) } @@ -438,13 +438,13 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router. } // update status phase - if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil { + if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil { c.recordEventWarningf(cd, "%v", err) return false } // notify - c.recorder.SetStatus(cd, flaggerv1.CanarySucceeded) + c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded) c.recordEventInfof(cd, "Promotion completed! Canary analysis was skipped for %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) c.sendNotification(cd, "Canary analysis was skipped, promotion finished.", @@ -454,7 +454,7 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router. } func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) { - if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryProgressing { + if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryPhaseProgressing { return true, nil } @@ -477,16 +477,16 @@ func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) { func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool) bool { c.recorder.SetStatus(cd, cd.Status.Phase) - if cd.Status.Phase == flaggerv1.CanaryProgressing { + if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing { return true } if cd.Status.Phase == "" { - if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryInitialized}); err != nil { + if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseInitialized}); err != nil { c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err) return false } - c.recorder.SetStatus(cd, flaggerv1.CanaryInitialized) + c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseInitialized) c.recordEventInfof(cd, "Initialization done! %s.%s", cd.Name, cd.Namespace) c.sendNotification(cd, "New deployment detected, initialization completed.", true, false) @@ -501,18 +501,18 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool) c.recordEventErrorf(cd, "%v", err) return false } - if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryProgressing}); err != nil { + if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseProgressing}); err != nil { c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err) return false } - c.recorder.SetStatus(cd, flaggerv1.CanaryProgressing) + c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseProgressing) return false } return false } func (c *Controller) hasCanaryRevisionChanged(cd *flaggerv1.Canary) bool { - if cd.Status.Phase == flaggerv1.CanaryProgressing { + if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing { if diff, _ := c.deployer.HasDeploymentChanged(cd); diff { return true } @@ -526,7 +526,7 @@ func (c *Controller) hasCanaryRevisionChanged(cd *flaggerv1.Canary) bool { func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool { for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks { if webhook.Type == flaggerv1.PreRolloutHook { - err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryProgressing, webhook) + err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook) if err != nil { c.recordEventWarningf(canary, "Halt %s.%s advancement pre-rollout check %s failed %v", canary.Name, canary.Namespace, webhook.Name, err) @@ -558,7 +558,7 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { // run external checks for _, webhook := range r.Spec.CanaryAnalysis.Webhooks { if webhook.Type == "" || webhook.Type == flaggerv1.RolloutHook { - err := CallWebhook(r.Name, r.Namespace, flaggerv1.CanaryProgressing, webhook) + err := CallWebhook(r.Name, r.Namespace, flaggerv1.CanaryPhaseProgressing, webhook) if err != nil { c.recordEventWarningf(r, "Halt %s.%s advancement external check %s failed %v", r.Name, r.Namespace, webhook.Name, err) diff --git a/pkg/controller/scheduler_test.go b/pkg/controller/scheduler_test.go index 32ba819d..bd88e8e4 100644 --- a/pkg/controller/scheduler_test.go +++ b/pkg/controller/scheduler_test.go @@ -47,7 +47,7 @@ func TestScheduler_Rollback(t *testing.T) { mocks.ctrl.advanceCanary("podinfo", "default", true) // update failed checks to max - err := mocks.deployer.SyncStatus(mocks.canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryProgressing, FailedChecks: 11}) + err := mocks.deployer.SyncStatus(mocks.canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryPhaseProgressing, FailedChecks: 11}) if err != nil { t.Fatal(err.Error()) } @@ -60,8 +60,8 @@ func TestScheduler_Rollback(t *testing.T) { t.Fatal(err.Error()) } - if c.Status.Phase != v1alpha3.CanaryFailed { - t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryFailed) + if c.Status.Phase != v1alpha3.CanaryPhaseFailed { + t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseFailed) } } @@ -101,8 +101,8 @@ func TestScheduler_SkipAnalysis(t *testing.T) { t.Errorf("Got skip analysis %v wanted %v", c.Spec.SkipAnalysis, true) } - if c.Status.Phase != v1alpha3.CanarySucceeded { - t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded) + if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded { + t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded) } } @@ -255,8 +255,8 @@ func TestScheduler_Promotion(t *testing.T) { t.Fatal(err.Error()) } - if c.Status.Phase != v1alpha3.CanarySucceeded { - t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded) + if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded { + t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded) } } @@ -326,8 +326,8 @@ func TestScheduler_ABTesting(t *testing.T) { t.Fatal(err.Error()) } - if c.Status.Phase != v1alpha3.CanarySucceeded { - t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded) + if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded { + t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded) } } diff --git a/pkg/controller/webhook_test.go b/pkg/controller/webhook_test.go index 528dc516..669a95c9 100644 --- a/pkg/controller/webhook_test.go +++ b/pkg/controller/webhook_test.go @@ -19,7 +19,7 @@ func TestCallWebhook(t *testing.T) { Metadata: &map[string]string{"key1": "val1"}, } - err := CallWebhook("podinfo", "default", flaggerv1.CanaryProgressing, hook) + err := CallWebhook("podinfo", "default", flaggerv1.CanaryPhaseProgressing, hook) if err != nil { t.Fatal(err.Error()) } @@ -35,7 +35,7 @@ func TestCallWebhook_StatusCode(t *testing.T) { URL: ts.URL, } - err := CallWebhook("podinfo", "default", flaggerv1.CanaryProgressing, hook) + err := CallWebhook("podinfo", "default", flaggerv1.CanaryPhaseProgressing, hook) if err == nil { t.Errorf("Got no error wanted %v", http.StatusInternalServerError) } diff --git a/pkg/metrics/recorder.go b/pkg/metrics/recorder.go index 6b8d50ed..d798bc65 100644 --- a/pkg/metrics/recorder.go +++ b/pkg/metrics/recorder.go @@ -87,9 +87,9 @@ func (cr *Recorder) SetTotal(namespace string, total int) { func (cr *Recorder) SetStatus(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) { status := 1 switch phase { - case flaggerv1.CanaryProgressing: + case flaggerv1.CanaryPhaseProgressing: status = 0 - case flaggerv1.CanaryFailed: + case flaggerv1.CanaryPhaseFailed: status = 2 default: status = 1 From 108bf9ca6514c507bbbaaaef5e419d5f635e0190 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 9 Jul 2019 17:10:43 +0300 Subject: [PATCH 03/10] Add initializing canary phase/status condition reason Fix HPA reconciliation min replicas diff --- go.sum | 7 +++++++ pkg/apis/flagger/v1alpha3/status.go | 2 ++ pkg/canary/deployer.go | 13 +++++++++++-- pkg/canary/status.go | 15 +++++++++------ pkg/controller/controller.go | 27 +++++++++++++++++++++------ pkg/controller/scheduler.go | 4 ++-- 6 files changed, 52 insertions(+), 16 deletions(-) diff --git a/go.sum b/go.sum index b000eb21..465cc6eb 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,7 @@ github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxU github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/avast/retry-go v2.2.0+incompatible h1:m+w7mVLWa/oKqX2xYqiEKQQkeGH8DDEXB/XnjS54Wyw= @@ -38,6 +39,7 @@ github.com/avast/retry-go v2.2.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevB github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= @@ -80,6 +82,7 @@ github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fgrosse/zaptest v1.1.0 h1:sK9hP0/xBoNX5qfFo3KWFluDXfc809APomI1QXuYELA= github.com/fgrosse/zaptest v1.1.0/go.mod h1:vMnRSul6kW7kIUXZgnZZcDwyTn8k49ODfAULL8nmL5w= @@ -192,6 +195,7 @@ github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= @@ -202,6 +206,7 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= @@ -265,6 +270,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -315,6 +321,7 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= diff --git a/pkg/apis/flagger/v1alpha3/status.go b/pkg/apis/flagger/v1alpha3/status.go index d28a860c..99afb980 100644 --- a/pkg/apis/flagger/v1alpha3/status.go +++ b/pkg/apis/flagger/v1alpha3/status.go @@ -38,6 +38,8 @@ type CanaryCondition struct { type CanaryPhase string const ( + // CanaryPhaseInitializing means the canary initializing is underway + CanaryPhaseInitializing CanaryPhase = "Initializing" // CanaryPhaseInitialized means the primary deployment, hpa and ClusterIP services // have been created along with the service mesh or ingress objects CanaryPhaseInitialized CanaryPhase = "Initialized" diff --git a/pkg/canary/deployer.go b/pkg/canary/deployer.go index aa83eab3..41ceed50 100644 --- a/pkg/canary/deployer.go +++ b/pkg/canary/deployer.go @@ -37,7 +37,7 @@ func (c *Deployer) Initialize(cd *flaggerv1.Canary, skipLivenessChecks bool) (la return "", ports, fmt.Errorf("creating deployment %s.%s failed: %v", primaryName, cd.Namespace, err) } - if cd.Status.Phase == "" { + if cd.Status.Phase == "" || cd.Status.Phase == flaggerv1.CanaryPhaseInitializing { if !skipLivenessChecks { _, readyErr := c.IsPrimaryReady(cd) if readyErr != nil { @@ -328,7 +328,8 @@ func (c *Deployer) reconcilePrimaryHpa(cd *flaggerv1.Canary, init bool) error { // update HPA if !init && primaryHpa != nil { diff := cmp.Diff(hpaSpec.Metrics, primaryHpa.Spec.Metrics) - if diff != "" || hpaSpec.MinReplicas != primaryHpa.Spec.MinReplicas || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas { + if diff != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas { + fmt.Println(diff, hpaSpec.MinReplicas, primaryHpa.Spec.MinReplicas, hpaSpec.MaxReplicas, primaryHpa.Spec.MaxReplicas) hpaClone := primaryHpa.DeepCopy() hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas @@ -425,3 +426,11 @@ func makePrimaryLabels(labels map[string]string, primaryName string, label strin func int32p(i int32) *int32 { return &i } + +func int32Default(i *int32) int32 { + if i == nil { + return 1 + } + + return *i +} diff --git a/pkg/canary/status.go b/pkg/canary/status.go index 163b0bee..ef98d438 100644 --- a/pkg/canary/status.go +++ b/pkg/canary/status.go @@ -39,7 +39,7 @@ func (c *Deployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatu cdCopy.Status.LastTransitionTime = metav1.Now() cdCopy.Status.TrackedConfigs = configs - if ok, conditions := c.makeStatusConditions(cd.Status, status.Phase); ok { + if ok, conditions := c.MakeStatusConditions(cd.Status, status.Phase); ok { cdCopy.Status.Conditions = conditions } @@ -100,7 +100,7 @@ func (c *Deployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPh cdCopy.Status.Iterations = 0 } - if ok, conditions := c.makeStatusConditions(cdCopy.Status, phase); ok { + if ok, conditions := c.MakeStatusConditions(cdCopy.Status, phase); ok { cdCopy.Status.Conditions = conditions } @@ -123,19 +123,22 @@ func (c *Deployer) getStatusCondition(status flaggerv1.CanaryStatus, conditionTy } // MakeStatusCondition updates the canary status conditions based on canary phase -func (c *Deployer) makeStatusConditions(canaryStatus flaggerv1.CanaryStatus, +func (c *Deployer) MakeStatusConditions(canaryStatus flaggerv1.CanaryStatus, phase flaggerv1.CanaryPhase) (bool, []flaggerv1.CanaryCondition) { currentCondition := c.getStatusCondition(canaryStatus, flaggerv1.PromotedType) message := "New deployment detected, starting initialization." status := corev1.ConditionUnknown switch phase { + case flaggerv1.CanaryPhaseInitializing: + status = corev1.ConditionUnknown + message = "New deployment detected, starting initialization." + case flaggerv1.CanaryPhaseInitialized: + status = corev1.ConditionTrue + message = "Deployment initialization completed." case flaggerv1.CanaryPhaseProgressing: status = corev1.ConditionUnknown message = "New revision detected, starting canary analysis." - case flaggerv1.CanaryPhaseInitialized: - status = corev1.ConditionTrue - message = "New deployment detected, initialization completed." case flaggerv1.CanaryPhaseSucceeded: status = corev1.ConditionTrue message = "Canary analysis completed successfully, promotion finished." diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 16a0b8a3..44d8870f 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -5,20 +5,21 @@ import ( "sync" "time" - "github.com/weaveworks/flagger/pkg/canary" - "github.com/weaveworks/flagger/pkg/metrics" - "github.com/weaveworks/flagger/pkg/router" - - "github.com/google/go-cmp/cmp" flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" + "github.com/weaveworks/flagger/pkg/canary" clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned" flaggerscheme "github.com/weaveworks/flagger/pkg/client/clientset/versioned/scheme" flaggerinformers "github.com/weaveworks/flagger/pkg/client/informers/externalversions/flagger/v1alpha3" flaggerlisters "github.com/weaveworks/flagger/pkg/client/listers/flagger/v1alpha3" + "github.com/weaveworks/flagger/pkg/metrics" "github.com/weaveworks/flagger/pkg/notifier" + "github.com/weaveworks/flagger/pkg/router" + + "github.com/google/go-cmp/cmp" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" @@ -211,10 +212,24 @@ func (c *Controller) syncHandler(key string) error { } cd, err := c.flaggerLister.Canaries(namespace).Get(name) if errors.IsNotFound(err) { - utilruntime.HandleError(fmt.Errorf("'%s' in work queue no longer exists", key)) + utilruntime.HandleError(fmt.Errorf("%s in work queue no longer exists", key)) return nil } + // set status condition for new canaries + if cd.Status.Conditions == nil { + if ok, conditions := c.deployer.MakeStatusConditions(cd.Status, flaggerv1.CanaryPhaseInitializing); ok { + cdCopy := cd.DeepCopy() + cdCopy.Status.Conditions = conditions + cdCopy.Status.LastTransitionTime = metav1.Now() + cdCopy.Status.Phase = flaggerv1.CanaryPhaseInitializing + _, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + if err != nil { + return fmt.Errorf("%s status condition update error: %v", key, err) + } + } + } + c.canaries.Store(fmt.Sprintf("%s.%s", cd.Name, cd.Namespace), cd) c.logger.Infof("Synced %s", key) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 10cbca22..de02d079 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -454,7 +454,7 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router. } func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) { - if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryPhaseProgressing { + if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryPhaseInitializing || cd.Status.Phase == flaggerv1.CanaryPhaseProgressing { return true, nil } @@ -481,7 +481,7 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool) return true } - if cd.Status.Phase == "" { + if cd.Status.Phase == "" || cd.Status.Phase == flaggerv1.CanaryPhaseInitializing { if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseInitialized}); err != nil { c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err) return false From afa2d079f67ebee8a618b9768568fba5e7ad6ad0 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 9 Jul 2019 17:11:13 +0300 Subject: [PATCH 04/10] Add status conditions and descriptions to CRD --- artifacts/flagger/crd.yaml | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index b1467d67..1b235577 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -102,17 +102,23 @@ spec: canaryAnalysis: properties: interval: + description: Canary schedule interval type: string pattern: "^[0-9]+(m|s)" iterations: + description: Number of checks to run for A/B Testing and Blue/Green type: number threshold: + description: Max number of failed checks before rollback type: number maxWeight: + description: Max traffic percentage routed to canary type: number stepWeight: + description: Canary incremental traffic percentage step type: number metrics: + description: Prometheus query list for this canary type: array properties: items: @@ -120,15 +126,20 @@ spec: required: ['name', 'threshold'] properties: name: + description: Name of the Prometheus metric type: string interval: + description: Interval of the promql query type: string pattern: "^[0-9]+(m|s)" threshold: + description: Max scalar value accepted for this metric type: number query: + description: Prometheus query type: string webhooks: + description: Webhook list for this canary type: array properties: items: @@ -136,8 +147,10 @@ spec: required: ['name', 'url', 'timeout'] properties: name: + description: Name of the webhook type: string type: + description: Type of the webhook pre, post or during rollout type: string enum: - "" @@ -145,28 +158,66 @@ spec: - rollout - post-rollout url: + description: URL address of this webhook type: string format: url timeout: + description: Request timeout for this webhook type: string pattern: "^[0-9]+(m|s)" status: properties: phase: + description: Analysis phase of this canary type: string enum: - "" + - Initializing - Initialized - Progressing - Succeeded - Failed canaryWeight: + description: Traffic weight percentage routed to canary type: number failedChecks: + description: Failed check count of the current canary analysis type: number iterations: + description: Iteration count of the current canary analysis type: number lastAppliedSpec: + description: LastAppliedSpec of this canary type: string lastTransitionTime: + description: LastTransitionTime of this canary + format: date-time type: string + conditions: + description: Status conditions of this canary + type: array + properties: + items: + type: object + required: ['type', 'status', 'reason'] + properties: + lastTransitionTime: + description: LastTransitionTime of this condition + format: date-time + type: string + lastUpdateTime: + description: LastUpdateTime of this condition + format: date-time + type: string + message: + description: Message associated with this condition + type: string + reason: + description: Reason for the current status of this condition + type: string + status: + description: Status of this condition + type: string + type: + description: Type of this condition + type: string From b26542f38da7ad3cd57f88f82999204506750ffc Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 10 Jul 2019 09:08:33 +0300 Subject: [PATCH 05/10] Do not trigger a canary deployment on manual rollback Save the primary spec hash and check if it matches the canary spec. If the canary hash is identical with the primary one skip promotion. --- pkg/apis/flagger/v1alpha3/status.go | 2 ++ pkg/canary/deployer.go | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/pkg/apis/flagger/v1alpha3/status.go b/pkg/apis/flagger/v1alpha3/status.go index 99afb980..5c43b0fd 100644 --- a/pkg/apis/flagger/v1alpha3/status.go +++ b/pkg/apis/flagger/v1alpha3/status.go @@ -64,6 +64,8 @@ type CanaryStatus struct { // +optional LastAppliedSpec string `json:"lastAppliedSpec,omitempty"` // +optional + LastPromotedSpec string `json:"lastPromotedSpec,omitempty"` + // +optional LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` // +optional Conditions []CanaryCondition `json:"conditions,omitempty"` diff --git a/pkg/canary/deployer.go b/pkg/canary/deployer.go index 41ceed50..1c5a47b5 100644 --- a/pkg/canary/deployer.go +++ b/pkg/canary/deployer.go @@ -113,12 +113,21 @@ func (c *Deployer) Promote(cd *flaggerv1.Canary) error { primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryName, label) + // apply update _, err = c.KubeClient.AppsV1().Deployments(cd.Namespace).Update(primaryCopy) if err != nil { return fmt.Errorf("updating deployment %s.%s template spec failed: %v", primaryCopy.GetName(), primaryCopy.Namespace, err) } + // update primary spec hash + cdClone := cd.DeepCopy() + cdClone.Status.LastPromotedSpec = cd.Status.LastAppliedSpec + _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdClone) + if err != nil { + return fmt.Errorf("updating canary status LastAppliedSpec failed: %v", err) + } + // update HPA if cd.Spec.AutoscalerRef != nil && cd.Spec.AutoscalerRef.Kind == "HorizontalPodAutoscaler" { if err := c.reconcilePrimaryHpa(cd, false); err != nil { @@ -149,6 +158,11 @@ func (c *Deployer) HasDeploymentChanged(cd *flaggerv1.Canary) (bool, error) { return false, fmt.Errorf("hash error %v", err) } + // do not trigger a canary deployment on manual rollback + if cd.Status.LastPromotedSpec == fmt.Sprintf("%d", newHash) { + return false, nil + } + if cd.Status.LastAppliedSpec != fmt.Sprintf("%d", newHash) { return true, nil } From caea00e47f3536f4450fbf9b11510a75279cc637 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 10 Jul 2019 09:42:49 +0300 Subject: [PATCH 06/10] Pin NGINX helm chart to version 1.8.2 --- test/e2e-nginx.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e-nginx.sh b/test/e2e-nginx.sh index e35c5a4d..027b063f 100755 --- a/test/e2e-nginx.sh +++ b/test/e2e-nginx.sh @@ -4,9 +4,10 @@ set -o errexit REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" +NGINX_VERSION=1.8.2 echo '>>> Installing NGINX Ingress' -helm upgrade -i nginx-ingress stable/nginx-ingress \ +helm upgrade -i nginx-ingress stable/nginx-ingress --version=${NGINX_VERSION} \ --wait \ --namespace ingress-nginx \ --set controller.stats.enabled=true \ From 9d89e0c83f496f102fb9abede14d93cb739e92bf Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 10 Jul 2019 09:55:20 +0300 Subject: [PATCH 07/10] Log status update error --- pkg/controller/controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 44d8870f..3e82f00d 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -225,6 +225,7 @@ func (c *Controller) syncHandler(key string) error { cdCopy.Status.Phase = flaggerv1.CanaryPhaseInitializing _, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) if err != nil { + c.logger.Errorf("%s status condition update error: %v", key, err) return fmt.Errorf("%s status condition update error: %v", key, err) } } From 9b6cfdeef7934f1575c6983f4ba0a6f13dc52ded Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 10 Jul 2019 09:55:46 +0300 Subject: [PATCH 08/10] Update Canary CRD helm chart and Kustomize --- charts/flagger/templates/crd.yaml | 51 +++++++++++++++++++++++++++++++ kustomize/base/flagger/crd.yaml | 51 +++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index a6eb7409..fcf326f1 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -103,17 +103,23 @@ spec: canaryAnalysis: properties: interval: + description: Canary schedule interval type: string pattern: "^[0-9]+(m|s)" iterations: + description: Number of checks to run for A/B Testing and Blue/Green type: number threshold: + description: Max number of failed checks before rollback type: number maxWeight: + description: Max traffic percentage routed to canary type: number stepWeight: + description: Canary incremental traffic percentage step type: number metrics: + description: Prometheus query list for this canary type: array properties: items: @@ -121,15 +127,20 @@ spec: required: ['name', 'threshold'] properties: name: + description: Name of the Prometheus metric type: string interval: + description: Interval of the promql query type: string pattern: "^[0-9]+(m|s)" threshold: + description: Max scalar value accepted for this metric type: number query: + description: Prometheus query type: string webhooks: + description: Webhook list for this canary type: array properties: items: @@ -137,8 +148,10 @@ spec: required: ['name', 'url', 'timeout'] properties: name: + description: Name of the webhook type: string type: + description: Type of the webhook pre, post or during rollout type: string enum: - "" @@ -146,29 +159,67 @@ spec: - rollout - post-rollout url: + description: URL address of this webhook type: string format: url timeout: + description: Request timeout for this webhook type: string pattern: "^[0-9]+(m|s)" status: properties: phase: + description: Analysis phase of this canary type: string enum: - "" + - Initializing - Initialized - Progressing - Succeeded - Failed canaryWeight: + description: Traffic weight percentage routed to canary type: number failedChecks: + description: Failed check count of the current canary analysis type: number iterations: + description: Iteration count of the current canary analysis type: number lastAppliedSpec: + description: LastAppliedSpec of this canary type: string lastTransitionTime: + description: LastTransitionTime of this canary + format: date-time type: string + conditions: + description: Status conditions of this canary + type: array + properties: + items: + type: object + required: ['type', 'status', 'reason'] + properties: + lastTransitionTime: + description: LastTransitionTime of this condition + format: date-time + type: string + lastUpdateTime: + description: LastUpdateTime of this condition + format: date-time + type: string + message: + description: Message associated with this condition + type: string + reason: + description: Reason for the current status of this condition + type: string + status: + description: Status of this condition + type: string + type: + description: Type of this condition + type: string {{- end }} diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index b1467d67..1b235577 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -102,17 +102,23 @@ spec: canaryAnalysis: properties: interval: + description: Canary schedule interval type: string pattern: "^[0-9]+(m|s)" iterations: + description: Number of checks to run for A/B Testing and Blue/Green type: number threshold: + description: Max number of failed checks before rollback type: number maxWeight: + description: Max traffic percentage routed to canary type: number stepWeight: + description: Canary incremental traffic percentage step type: number metrics: + description: Prometheus query list for this canary type: array properties: items: @@ -120,15 +126,20 @@ spec: required: ['name', 'threshold'] properties: name: + description: Name of the Prometheus metric type: string interval: + description: Interval of the promql query type: string pattern: "^[0-9]+(m|s)" threshold: + description: Max scalar value accepted for this metric type: number query: + description: Prometheus query type: string webhooks: + description: Webhook list for this canary type: array properties: items: @@ -136,8 +147,10 @@ spec: required: ['name', 'url', 'timeout'] properties: name: + description: Name of the webhook type: string type: + description: Type of the webhook pre, post or during rollout type: string enum: - "" @@ -145,28 +158,66 @@ spec: - rollout - post-rollout url: + description: URL address of this webhook type: string format: url timeout: + description: Request timeout for this webhook type: string pattern: "^[0-9]+(m|s)" status: properties: phase: + description: Analysis phase of this canary type: string enum: - "" + - Initializing - Initialized - Progressing - Succeeded - Failed canaryWeight: + description: Traffic weight percentage routed to canary type: number failedChecks: + description: Failed check count of the current canary analysis type: number iterations: + description: Iteration count of the current canary analysis type: number lastAppliedSpec: + description: LastAppliedSpec of this canary type: string lastTransitionTime: + description: LastTransitionTime of this canary + format: date-time type: string + conditions: + description: Status conditions of this canary + type: array + properties: + items: + type: object + required: ['type', 'status', 'reason'] + properties: + lastTransitionTime: + description: LastTransitionTime of this condition + format: date-time + type: string + lastUpdateTime: + description: LastUpdateTime of this condition + format: date-time + type: string + message: + description: Message associated with this condition + type: string + reason: + description: Reason for the current status of this condition + type: string + status: + description: Status of this condition + type: string + type: + description: Type of this condition + type: string From ff4aa6206129a6b32498849bef7afcefac1c5755 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 10 Jul 2019 11:31:20 +0300 Subject: [PATCH 09/10] Retry canary status update on conflict --- go.mod | 1 + pkg/canary/deployer.go | 8 --- pkg/canary/status.go | 155 +++++++++++++++++++++++++++++------------ 3 files changed, 113 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index 7100be67..59b27495 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/hashstructure v1.0.0 + github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect github.com/prometheus/common v0.3.0 // indirect diff --git a/pkg/canary/deployer.go b/pkg/canary/deployer.go index 1c5a47b5..070b72b1 100644 --- a/pkg/canary/deployer.go +++ b/pkg/canary/deployer.go @@ -120,14 +120,6 @@ func (c *Deployer) Promote(cd *flaggerv1.Canary) error { primaryCopy.GetName(), primaryCopy.Namespace, err) } - // update primary spec hash - cdClone := cd.DeepCopy() - cdClone.Status.LastPromotedSpec = cd.Status.LastAppliedSpec - _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdClone) - if err != nil { - return fmt.Errorf("updating canary status LastAppliedSpec failed: %v", err) - } - // update HPA if cd.Spec.AutoscalerRef != nil && cd.Spec.AutoscalerRef.Kind == "HorizontalPodAutoscaler" { if err := c.reconcilePrimaryHpa(cd, false); err != nil { diff --git a/pkg/canary/status.go b/pkg/canary/status.go index ef98d438..2001470c 100644 --- a/pkg/canary/status.go +++ b/pkg/canary/status.go @@ -2,8 +2,10 @@ package canary import ( "fmt" + "k8s.io/client-go/util/retry" "github.com/mitchellh/hashstructure" + ex "github.com/pkg/errors" flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -17,96 +19,163 @@ func (c *Deployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatu if errors.IsNotFound(err) { return fmt.Errorf("deployment %s.%s not found", cd.Spec.TargetRef.Name, cd.Namespace) } - return fmt.Errorf("deployment %s.%s query error %v", cd.Spec.TargetRef.Name, cd.Namespace, err) + return ex.Wrap(err, "SyncStatus deployment query error") } configs, err := c.ConfigTracker.GetConfigRefs(cd) if err != nil { - return fmt.Errorf("configs query error %v", err) + return ex.Wrap(err, "SyncStatus configs query error") } hash, err := hashstructure.Hash(dep.Spec.Template, nil) if err != nil { - return fmt.Errorf("hash error %v", err) + return ex.Wrap(err, "SyncStatus hash error") } - cdCopy := cd.DeepCopy() - cdCopy.Status.Phase = status.Phase - cdCopy.Status.CanaryWeight = status.CanaryWeight - cdCopy.Status.FailedChecks = status.FailedChecks - cdCopy.Status.Iterations = status.Iterations - cdCopy.Status.LastAppliedSpec = fmt.Sprintf("%d", hash) - cdCopy.Status.LastTransitionTime = metav1.Now() - cdCopy.Status.TrackedConfigs = configs + firstTry := true + err = retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + var selErr error + if !firstTry { + cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{}) + if selErr != nil { + return selErr + } + } + cdCopy := cd.DeepCopy() + cdCopy.Status.Phase = status.Phase + cdCopy.Status.CanaryWeight = status.CanaryWeight + cdCopy.Status.FailedChecks = status.FailedChecks + cdCopy.Status.Iterations = status.Iterations + cdCopy.Status.LastAppliedSpec = fmt.Sprintf("%d", hash) + cdCopy.Status.LastTransitionTime = metav1.Now() + cdCopy.Status.TrackedConfigs = configs - if ok, conditions := c.MakeStatusConditions(cd.Status, status.Phase); ok { - cdCopy.Status.Conditions = conditions - } + if ok, conditions := c.MakeStatusConditions(cd.Status, status.Phase); ok { + cdCopy.Status.Conditions = conditions + } - cd, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + firstTry = false + return + }) if err != nil { - return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + return ex.Wrap(err, "SyncStatus") } return nil } // SetStatusFailedChecks updates the canary failed checks counter func (c *Deployer) SetStatusFailedChecks(cd *flaggerv1.Canary, val int) error { - cdCopy := cd.DeepCopy() - cdCopy.Status.FailedChecks = val - cdCopy.Status.LastTransitionTime = metav1.Now() + firstTry := true + err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + var selErr error + if !firstTry { + cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{}) + if selErr != nil { + return selErr + } + } + cdCopy := cd.DeepCopy() + cdCopy.Status.FailedChecks = val + cdCopy.Status.LastTransitionTime = metav1.Now() - cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + firstTry = false + return + }) if err != nil { - return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + return ex.Wrap(err, "SetStatusFailedChecks") } return nil } // SetStatusWeight updates the canary status weight value func (c *Deployer) SetStatusWeight(cd *flaggerv1.Canary, val int) error { - cdCopy := cd.DeepCopy() - cdCopy.Status.CanaryWeight = val - cdCopy.Status.LastTransitionTime = metav1.Now() + firstTry := true + err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + var selErr error + if !firstTry { + cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{}) + if selErr != nil { + return selErr + } + } + cdCopy := cd.DeepCopy() + cdCopy.Status.CanaryWeight = val + cdCopy.Status.LastTransitionTime = metav1.Now() - cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + firstTry = false + return + }) if err != nil { - return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + return ex.Wrap(err, "SetStatusWeight") } return nil } // SetStatusIterations updates the canary status iterations value func (c *Deployer) SetStatusIterations(cd *flaggerv1.Canary, val int) error { - cdCopy := cd.DeepCopy() - cdCopy.Status.Iterations = val - cdCopy.Status.LastTransitionTime = metav1.Now() + firstTry := true + err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + var selErr error + if !firstTry { + cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{}) + if selErr != nil { + return selErr + } + } + + cdCopy := cd.DeepCopy() + cdCopy.Status.Iterations = val + cdCopy.Status.LastTransitionTime = metav1.Now() + + _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + firstTry = false + return + }) - cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) if err != nil { - return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + return ex.Wrap(err, "SetStatusIterations") } return nil } // SetStatusPhase updates the canary status phase func (c *Deployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) error { - cdCopy := cd.DeepCopy() - cdCopy.Status.Phase = phase - cdCopy.Status.LastTransitionTime = metav1.Now() + firstTry := true + err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + var selErr error + if !firstTry { + cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{}) + if selErr != nil { + return selErr + } + } + cdCopy := cd.DeepCopy() + cdCopy.Status.Phase = phase + cdCopy.Status.LastTransitionTime = metav1.Now() - if phase != flaggerv1.CanaryPhaseProgressing { - cdCopy.Status.CanaryWeight = 0 - cdCopy.Status.Iterations = 0 - } + if phase != flaggerv1.CanaryPhaseProgressing { + cdCopy.Status.CanaryWeight = 0 + cdCopy.Status.Iterations = 0 + } - if ok, conditions := c.MakeStatusConditions(cdCopy.Status, phase); ok { - cdCopy.Status.Conditions = conditions - } + // on promotion set primary spec hash + if phase == flaggerv1.CanaryPhaseInitialized || phase == flaggerv1.CanaryPhaseSucceeded { + cdCopy.Status.LastPromotedSpec = cd.Status.LastAppliedSpec + } - cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + if ok, conditions := c.MakeStatusConditions(cdCopy.Status, phase); ok { + cdCopy.Status.Conditions = conditions + } + + _, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + firstTry = false + return + }) if err != nil { - return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + return ex.Wrap(err, "SetStatusPhase") } return nil } From 3786a49f00bca1bcc7e841abb569287cb9fdb42e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 16 Jul 2019 11:20:42 +0200 Subject: [PATCH 10/10] Update Linkerd e2e to v2.4.0 --- test/e2e-linkerd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-linkerd.sh b/test/e2e-linkerd.sh index 5dfa94f5..15db6ea2 100755 --- a/test/e2e-linkerd.sh +++ b/test/e2e-linkerd.sh @@ -2,7 +2,7 @@ set -o errexit -LINKERD_VER="edge-19.6.4" +LINKERD_VER="stable-2.4.0" REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"