diff --git a/README.md b/README.md index 3e727349..bc2fce37 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. -The canary analysis can be extended with webhooks for running integration tests, load tests or any other custom -validation. +The canary analysis can be extended with webhooks for running integration tests, +load tests or any other custom validation. ### Install @@ -28,7 +28,7 @@ helm upgrade -i flagger flagger/flagger \ --set metricsServer=http://prometheus.istio-system:9090 ``` -Flagger is compatible with Kubernetes >1.10.0 and Istio >1.0.0. +Flagger is compatible with Kubernetes >1.11.0 and Istio >1.0.0. ### Usage @@ -242,15 +242,16 @@ kubectl -n test set image deployment/podinfo \ podinfod=quay.io/stefanprodan/podinfo:1.2.1 ``` -Flagger detects that the deployment revision changed and starts a new rollout: +Flagger detects that the deployment revision changed and starts a new canary analysis: ``` kubectl -n test describe canary/podinfo Status: - Canary Revision: 19871136 - Failed Checks: 0 - State: finished + Canary Weight: 0 + Failed Checks: 0 + Last Transition Time: 2019-01-16T13:47:16Z + Phase: Succeeded Events: Type Reason Age From Message ---- ------ ---- ---- ------- @@ -272,6 +273,15 @@ Events: Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test ``` +You can monitor all canaries with: + +```bash +watch kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 5 2019-01-16T14:05:07Z +``` + During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout. Create a tester pod and exec into it: @@ -300,9 +310,10 @@ the canary is scaled to zero and the rollout is marked as failed. kubectl -n test describe canary/podinfo Status: - Canary Revision: 16695041 - Failed Checks: 10 - State: failed + Canary Weight: 0 + Failed Checks: 10 + Last Transition Time: 2019-01-16T13:47:16Z + Phase: Failed Events: Type Reason Age From Message ---- ------ ---- ---- ------- diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 9acfa09b..e674913c 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -19,7 +19,21 @@ spec: plural: canaries singular: canary kind: Canary + categories: + - all scope: Namespaced + subresources: + status: {} + additionalPrinterColumns: + - name: Status + type: string + JSONPath: .status.phase + - name: Weight + type: string + JSONPath: .status.canaryWeight + - name: LastTransitionTime + type: string + JSONPath: .status.lastTransitionTime validation: openAPIV3Schema: properties: diff --git a/charts/flagger/Chart.yaml b/charts/flagger/Chart.yaml index eb6ac91a..77788dd9 100644 --- a/charts/flagger/Chart.yaml +++ b/charts/flagger/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 name: flagger -version: 0.3.0 +version: 0.4.0 appVersion: 0.3.1-alpha.1 -kubeVersion: ">=1.9.0-0" +kubeVersion: ">=1.11.0-0" engine: gotpl description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. home: https://docs.flagger.app diff --git a/charts/flagger/README.md b/charts/flagger/README.md index 96fd2d92..2a541a4b 100644 --- a/charts/flagger/README.md +++ b/charts/flagger/README.md @@ -8,7 +8,7 @@ Based on the KPIs analysis a canary is promoted or aborted and the analysis resu ## Prerequisites -* Kubernetes >= 1.9 +* Kubernetes >= 1.11 * Istio >= 1.0 * Prometheus >= 2.6 diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index c9789f0f..22e80bc9 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -20,7 +20,21 @@ spec: plural: canaries singular: canary kind: Canary + categories: + - all scope: Namespaced + subresources: + status: {} + additionalPrinterColumns: + - name: Status + type: string + JSONPath: .status.phase + - name: Weight + type: string + JSONPath: .status.canaryWeight + - name: LastTransitionTime + type: string + JSONPath: .status.lastTransitionTime validation: openAPIV3Schema: properties: diff --git a/docs/gitbook/README.md b/docs/gitbook/README.md index 44828148..92dba1e3 100644 --- a/docs/gitbook/README.md +++ b/docs/gitbook/README.md @@ -4,13 +4,20 @@ description: Flagger is an Istio progressive delivery Kubernetes operator # Introduction -[Flagger](https://github.com/stefanprodan/flagger) is a **Kubernetes** operator that automates the promotion of canary deployments using **Istio** routing for traffic shifting and **Prometheus** metrics for canary analysis. +[Flagger](https://github.com/stefanprodan/flagger) is a **Kubernetes** operator that automates the promotion of canary +deployments using **Istio** routing for traffic shifting and **Prometheus** metrics for canary analysis. +The canary analysis can be extended with webhooks for running integration tests, +load tests or any other custom validation. -Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators like HTTP requests success rate, requests average duration and pods health. Based on the **KPIs** analysis a canary is promoted or aborted and the analysis result is published to **Slack**. +Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance +indicators like HTTP requests success rate, requests average duration and pods health. +Based on the **KPIs** analysis a canary is promoted or aborted and the analysis result is published to **Slack**. ![Flagger overview diagram](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-canary-overview.png) -Flagger can be configured with Kubernetes custom resources \(canaries.flagger.app kind\) and is compatible with any CI/CD solutions made for Kubernetes. Since Flagger is declarative and reacts to Kubernetes events, it can be used in **GitOps** pipelines together with Weave Flux or JenkinsX. +Flagger can be configured with Kubernetes custom resources \(canaries.flagger.app kind\) and is compatible with +any CI/CD solutions made for Kubernetes. Since Flagger is declarative and reacts to Kubernetes events, +it can be used in **GitOps** pipelines together with Weave Flux or JenkinsX. This project is sponsored by [Weaveworks](https://www.weave.works/) diff --git a/docs/gitbook/install/install-flagger.md b/docs/gitbook/install/install-flagger.md index 2b8c4d76..fd85d34d 100644 --- a/docs/gitbook/install/install-flagger.md +++ b/docs/gitbook/install/install-flagger.md @@ -6,7 +6,7 @@ If you are new to Istio you can follow this GKE guide **Prerequisites** -* Kubernetes >= 1.9 +* Kubernetes >= 1.11 * Istio >= 1.0 * Prometheus >= 2.6 diff --git a/docs/gitbook/usage/progressive-delivery.md b/docs/gitbook/usage/progressive-delivery.md index 71455a08..d2805897 100644 --- a/docs/gitbook/usage/progressive-delivery.md +++ b/docs/gitbook/usage/progressive-delivery.md @@ -108,9 +108,9 @@ Flagger detects that the deployment revision changed and starts a new rollout: kubectl -n test describe canary/podinfo Status: - Canary Revision: 19871136 - Failed Checks: 0 - State: finished + Canary Weight: 0 + Failed Checks: 0 + Phase: Succeeded Events: Type Reason Age From Message ---- ------ ---- ---- ------- @@ -132,6 +132,17 @@ Events: Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test ``` +You can monitor all canaries with: + +```bash +watch kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 15 2019-01-16T14:05:07Z +prod frontend Succeeded 0 2019-01-15T16:15:07Z +prod backend Failed 0 2019-01-14T17:05:07Z +``` + During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout. Create a tester pod and exec into it: @@ -162,9 +173,9 @@ When the number of failed checks reaches the canary analysis threshold, the traf kubectl -n test describe canary/podinfo Status: - Canary Revision: 16695041 - Failed Checks: 10 - State: failed + Canary Weight: 0 + Failed Checks: 10 + Phase: Failed Events: Type Reason Age From Message ---- ------ ---- ---- ------- @@ -181,5 +192,3 @@ Events: Warning Synced 1m flagger Canary failed! Scaling down podinfo.test ``` -#### - diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index 944f9efa..c356d8a0 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -56,7 +56,7 @@ type CanarySpec struct { CanaryAnalysis CanaryAnalysis `json:"canaryAnalysis"` // the maximum time in seconds for a canary deployment to make progress - // before it is considered to be failed. Defaults to 60s. + // before it is considered to be failed. Defaults to ten minutes. ProgressDeadlineSeconds *int32 `json:"progressDeadlineSeconds,omitempty"` } @@ -70,21 +70,30 @@ type CanaryList struct { Items []Canary `json:"items"` } -// CanaryState used for status state op -type CanaryState string +// CanaryPhase is a label for the condition of a canary at the current time +type CanaryPhase string const ( - CanaryRunning CanaryState = "running" - CanaryFinished CanaryState = "finished" - CanaryFailed CanaryState = "failed" - CanaryInitialized CanaryState = "initialized" + // CanaryInitialized means the primary deployment, hpa and ClusterIP services + // have been created along with the Istio virtual service + CanaryInitialized CanaryPhase = "Initialized" + // CanaryProgressing means the canary analysis is underway + CanaryProgressing CanaryPhase = "Progressing" + // CanarySucceeded means the canary analysis has been successful + // and the canary deployment has been promoted + CanarySucceeded CanaryPhase = "Succeeded" + // CanaryFailed means the canary analysis failed + // and the canary deployment has been scaled to zero + CanaryFailed CanaryPhase = "Failed" ) // CanaryStatus is used for state persistence (read-only) type CanaryStatus struct { - State CanaryState `json:"state"` - CanaryRevision string `json:"canaryRevision"` - FailedChecks int `json:"failedChecks"` + Phase CanaryPhase `json:"phase"` + FailedChecks int `json:"failedChecks"` + CanaryWeight int `json:"canaryWeight"` + // +optional + LastAppliedSpec string `json:"lastAppliedSpec,omitempty"` // +optional LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` } @@ -139,6 +148,7 @@ func (c *Canary) GetProgressDeadlineSeconds() int { return ProgressDeadlineSeconds } +// GetAnalysisInterval returns the canary analysis interval (default 60s) func (c *Canary) GetAnalysisInterval() time.Duration { if c.Spec.CanaryAnalysis.Interval == "" { return AnalysisInterval diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index fd494f17..b6990df9 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -248,17 +248,17 @@ func checkCustomResourceType(obj interface{}, logger *zap.SugaredLogger) (flagge } func (c *Controller) recordEventInfof(r *flaggerv1.Canary, template string, args ...interface{}) { - c.logger.Infof(template, args...) + c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Infof(template, args...) c.eventRecorder.Event(r, corev1.EventTypeNormal, "Synced", fmt.Sprintf(template, args...)) } func (c *Controller) recordEventErrorf(r *flaggerv1.Canary, template string, args ...interface{}) { - c.logger.Errorf(template, args...) + c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Errorf(template, args...) c.eventRecorder.Event(r, corev1.EventTypeWarning, "Synced", fmt.Sprintf(template, args...)) } func (c *Controller) recordEventWarningf(r *flaggerv1.Canary, template string, args ...interface{}) { - c.logger.Infof(template, args...) + c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Infof(template, args...) c.eventRecorder.Event(r, corev1.EventTypeWarning, "Synced", fmt.Sprintf(template, args...)) } diff --git a/pkg/controller/deployer.go b/pkg/controller/deployer.go index f590c781..a5d65a3e 100644 --- a/pkg/controller/deployer.go +++ b/pkg/controller/deployer.go @@ -133,14 +133,14 @@ func (c *CanaryDeployer) IsNewSpec(cd *flaggerv1.Canary) (bool, error) { return false, fmt.Errorf("deployment %s.%s query error %v", targetName, cd.Namespace, err) } - if cd.Status.CanaryRevision == "" { + if cd.Status.LastAppliedSpec == "" { return true, nil } newSpec := &canary.Spec.Template.Spec - oldSpecJson, err := base64.StdEncoding.DecodeString(cd.Status.CanaryRevision) + oldSpecJson, err := base64.StdEncoding.DecodeString(cd.Status.LastAppliedSpec) if err != nil { - return false, err + return false, fmt.Errorf("%s.%s decode error %v", cd.Name, cd.Namespace, err) } oldSpec := &corev1.PodSpec{} err = json.Unmarshal(oldSpecJson, oldSpec) @@ -156,26 +156,51 @@ func (c *CanaryDeployer) IsNewSpec(cd *flaggerv1.Canary) (bool, error) { return false, nil } -// SetFailedChecks updates the canary failed checks counter -func (c *CanaryDeployer) SetFailedChecks(cd *flaggerv1.Canary, val int) error { +// ShouldAdvance determines if the canary analysis can proceed +func (c *CanaryDeployer) ShouldAdvance(cd *flaggerv1.Canary) (bool, error) { + if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryProgressing { + return true, nil + } + return c.IsNewSpec(cd) +} + +// SetStatusFailedChecks updates the canary failed checks counter +func (c *CanaryDeployer) SetStatusFailedChecks(cd *flaggerv1.Canary, val int) error { cdCopy := cd.DeepCopy() cdCopy.Status.FailedChecks = val cdCopy.Status.LastTransitionTime = metav1.Now() - cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Update(cdCopy) + cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) if err != nil { return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) } return nil } -// SetState updates the canary status state -func (c *CanaryDeployer) SetState(cd *flaggerv1.Canary, state flaggerv1.CanaryState) error { +// SetStatusWeight updates the canary status weight value +func (c *CanaryDeployer) SetStatusWeight(cd *flaggerv1.Canary, val int) error { cdCopy := cd.DeepCopy() - cdCopy.Status.State = state + cdCopy.Status.CanaryWeight = val cdCopy.Status.LastTransitionTime = metav1.Now() - cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Update(cdCopy) + cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) + if err != nil { + return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) + } + return nil +} + +// SetStatusPhase updates the canary status phase +func (c *CanaryDeployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) error { + cdCopy := cd.DeepCopy() + cdCopy.Status.Phase = phase + cdCopy.Status.LastTransitionTime = metav1.Now() + + if phase != flaggerv1.CanaryProgressing { + cdCopy.Status.CanaryWeight = 0 + } + + cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) if err != nil { return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) } @@ -198,12 +223,13 @@ func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.Canar } cdCopy := cd.DeepCopy() - cdCopy.Status.State = status.State + cdCopy.Status.Phase = status.Phase + cdCopy.Status.CanaryWeight = status.CanaryWeight cdCopy.Status.FailedChecks = status.FailedChecks - cdCopy.Status.CanaryRevision = base64.StdEncoding.EncodeToString(specJson) + cdCopy.Status.LastAppliedSpec = base64.StdEncoding.EncodeToString(specJson) cdCopy.Status.LastTransitionTime = metav1.Now() - cd, err = c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Update(cdCopy) + cd, err = c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) if err != nil { return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err) } @@ -239,8 +265,8 @@ func (c *CanaryDeployer) Sync(cd *flaggerv1.Canary) error { return fmt.Errorf("creating deployment %s.%s failed: %v", primaryName, cd.Namespace, err) } - if cd.Status.State == "" { - c.logger.Infof("Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) + if cd.Status.Phase == "" { + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) if err := c.Scale(cd, 0); err != nil { return err } @@ -307,7 +333,7 @@ func (c *CanaryDeployer) createPrimaryDeployment(cd *flaggerv1.Canary) error { return err } - c.logger.Infof("Deployment %s.%s created", primaryDep.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Deployment %s.%s created", primaryDep.GetName(), cd.Namespace) } return nil @@ -355,7 +381,7 @@ func (c *CanaryDeployer) createPrimaryHpa(cd *flaggerv1.Canary) error { if err != nil { return err } - c.logger.Infof("HorizontalPodAutoscaler %s.%s created", primaryHpa.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("HorizontalPodAutoscaler %s.%s created", primaryHpa.GetName(), cd.Namespace) } return nil diff --git a/pkg/controller/deployer_test.go b/pkg/controller/deployer_test.go index 2f9ea9ea..69910efc 100644 --- a/pkg/controller/deployer_test.go +++ b/pkg/controller/deployer_test.go @@ -351,7 +351,7 @@ func TestCanaryDeployer_SetFailedChecks(t *testing.T) { t.Fatal(err.Error()) } - err = deployer.SetFailedChecks(canary, 1) + err = deployer.SetStatusFailedChecks(canary, 1) if err != nil { t.Fatal(err.Error()) } @@ -387,7 +387,7 @@ func TestCanaryDeployer_SetState(t *testing.T) { t.Fatal(err.Error()) } - err = deployer.SetState(canary, v1alpha3.CanaryRunning) + err = deployer.SetStatusPhase(canary, v1alpha3.CanaryProgressing) if err != nil { t.Fatal(err.Error()) } @@ -397,8 +397,8 @@ func TestCanaryDeployer_SetState(t *testing.T) { t.Fatal(err.Error()) } - if res.Status.State != v1alpha3.CanaryRunning { - t.Errorf("Got %v wanted %v", res.Status.State, v1alpha3.CanaryRunning) + if res.Status.Phase != v1alpha3.CanaryProgressing { + t.Errorf("Got %v wanted %v", res.Status.Phase, v1alpha3.CanaryProgressing) } } @@ -424,7 +424,7 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) { } status := v1alpha3.CanaryStatus{ - State: v1alpha3.CanaryRunning, + Phase: v1alpha3.CanaryProgressing, FailedChecks: 2, } err = deployer.SyncStatus(canary, status) @@ -437,8 +437,8 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) { t.Fatal(err.Error()) } - if res.Status.State != status.State { - t.Errorf("Got state %v wanted %v", res.Status.State, status.State) + if res.Status.Phase != status.Phase { + t.Errorf("Got state %v wanted %v", res.Status.Phase, status.Phase) } if res.Status.FailedChecks != status.FailedChecks { diff --git a/pkg/controller/recorder.go b/pkg/controller/recorder.go index 31b2444e..47e23a6e 100644 --- a/pkg/controller/recorder.go +++ b/pkg/controller/recorder.go @@ -72,8 +72,8 @@ func (cr *CanaryRecorder) SetTotal(namespace string, total int) { // SetStatus sets the last known canary analysis status func (cr *CanaryRecorder) SetStatus(cd *flaggerv1.Canary) { status := 1 - switch cd.Status.State { - case flaggerv1.CanaryRunning: + switch cd.Status.Phase { + case flaggerv1.CanaryProgressing: status = 0 case flaggerv1.CanaryFailed: status = 2 diff --git a/pkg/controller/router.go b/pkg/controller/router.go index ce8f787b..2c5e130f 100644 --- a/pkg/controller/router.go +++ b/pkg/controller/router.go @@ -79,7 +79,7 @@ func (c *CanaryRouter) createServices(cd *flaggerv1.Canary) error { if err != nil { return err } - c.logger.Infof("Service %s.%s created", canaryService.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Service %s.%s created", canaryService.GetName(), cd.Namespace) } canaryTestServiceName := fmt.Sprintf("%s-canary", cd.Spec.TargetRef.Name) @@ -118,7 +118,7 @@ func (c *CanaryRouter) createServices(cd *flaggerv1.Canary) error { if err != nil { return err } - c.logger.Infof("Service %s.%s created", canaryTestService.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Service %s.%s created", canaryTestService.GetName(), cd.Namespace) } primaryService, err := c.kubeClient.CoreV1().Services(cd.Namespace).Get(primaryName, metav1.GetOptions{}) @@ -157,7 +157,7 @@ func (c *CanaryRouter) createServices(cd *flaggerv1.Canary) error { return err } - c.logger.Infof("Service %s.%s created", primaryService.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Service %s.%s created", primaryService.GetName(), cd.Namespace) } return nil @@ -218,7 +218,7 @@ func (c *CanaryRouter) createVirtualService(cd *flaggerv1.Canary) error { if err != nil { return fmt.Errorf("VirtualService %s.%s create error %v", targetName, cd.Namespace, err) } - c.logger.Infof("VirtualService %s.%s created", virtualService.GetName(), cd.Namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("VirtualService %s.%s created", virtualService.GetName(), cd.Namespace) } return nil @@ -276,13 +276,15 @@ func (c *CanaryRouter) SetRoutes( } return fmt.Errorf("VirtualService %s.%s query error %v", targetName, cd.Namespace, err) } - vs.Spec.Http = []istiov1alpha3.HTTPRoute{ + + vsCopy := vs.DeepCopy() + vsCopy.Spec.Http = []istiov1alpha3.HTTPRoute{ { Route: []istiov1alpha3.DestinationWeight{primary, canary}, }, } - vs, err = c.istioClient.NetworkingV1alpha3().VirtualServices(cd.Namespace).Update(vs) + vs, err = c.istioClient.NetworkingV1alpha3().VirtualServices(cd.Namespace).Update(vsCopy) if err != nil { return fmt.Errorf("VirtualService %s.%s update failed: %v", targetName, cd.Namespace, err) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 25a95892..f6975df3 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -75,7 +75,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { // check if the canary exists cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(namespace).Get(name, v1.GetOptions{}) if err != nil { - c.logger.Errorf("Canary %s.%s not found", name, namespace) + c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)).Errorf("Canary %s.%s not found", name, namespace) return } @@ -91,6 +91,13 @@ func (c *Controller) advanceCanary(name string, namespace string) { return } + if ok, err := c.deployer.ShouldAdvance(cd); !ok { + if err != nil { + c.recordEventWarningf(cd, "%v", err) + } + return + } + // set max weight default value to 100% maxWeight := 100 if cd.Spec.CanaryAnalysis.MaxWeight > 0 { @@ -130,7 +137,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // check if the number of failed checks reached the threshold - if cd.Status.State == flaggerv1.CanaryRunning && + if cd.Status.Phase == flaggerv1.CanaryProgressing && (!retriable || cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold) { if cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold { @@ -166,8 +173,8 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // mark canary as failed - if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{State: flaggerv1.CanaryFailed}); err != nil { - c.logger.Errorf("%v", err) + if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed, CanaryWeight: 0}); err != nil { + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err) return } @@ -181,7 +188,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { c.recordEventInfof(cd, "Starting canary deployment for %s.%s", cd.Name, cd.Namespace) } else { if ok := c.analyseCanary(cd); !ok { - if err := c.deployer.SetFailedChecks(cd, cd.Status.FailedChecks+1); err != nil { + if err := c.deployer.SetStatusFailedChecks(cd, cd.Status.FailedChecks+1); err != nil { c.recordEventWarningf(cd, "%v", err) return } @@ -205,6 +212,12 @@ func (c *Controller) advanceCanary(name string, namespace string) { return } + // update weight status + if err := c.deployer.SetStatusWeight(cd, canaryRoute.Weight); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } + c.recorder.SetWeight(cd, primaryRoute.Weight, canaryRoute.Weight) c.recordEventInfof(cd, "Advance %s.%s canary weight %v", cd.Name, cd.Namespace, canaryRoute.Weight) @@ -236,8 +249,8 @@ func (c *Controller) advanceCanary(name string, namespace string) { return } - // update status - if err := c.deployer.SetState(cd, flaggerv1.CanaryFinished); err != nil { + // update status phase + if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil { c.recordEventWarningf(cd, "%v", err) return } @@ -249,13 +262,13 @@ func (c *Controller) advanceCanary(name string, namespace string) { func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDeployer) bool { c.recorder.SetStatus(cd) - if cd.Status.State == "running" { + if cd.Status.Phase == flaggerv1.CanaryProgressing { return true } - if cd.Status.State == "" { - if err := deployer.SyncStatus(cd, flaggerv1.CanaryStatus{State: flaggerv1.CanaryInitialized}); err != nil { - c.logger.Errorf("%v", err) + if cd.Status.Phase == "" { + if err := deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryInitialized}); err != nil { + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err) return false } c.recorder.SetStatus(cd) @@ -273,8 +286,8 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDepl c.recordEventErrorf(cd, "%v", err) return false } - if err := deployer.SyncStatus(cd, flaggerv1.CanaryStatus{State: flaggerv1.CanaryRunning}); err != nil { - c.logger.Errorf("%v", err) + if err := deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryProgressing}); err != nil { + c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err) return false } c.recorder.SetStatus(cd) diff --git a/pkg/controller/scheduler_test.go b/pkg/controller/scheduler_test.go index e764f9b4..b796cf5f 100644 --- a/pkg/controller/scheduler_test.go +++ b/pkg/controller/scheduler_test.go @@ -194,7 +194,7 @@ func TestScheduler_Rollback(t *testing.T) { ctrl.advanceCanary("podinfo", "default") // update failed checks to max - err := deployer.SyncStatus(canary, v1alpha3.CanaryStatus{State: v1alpha3.CanaryRunning, FailedChecks: 11}) + err := deployer.SyncStatus(canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryProgressing, FailedChecks: 11}) if err != nil { t.Fatal(err.Error()) } @@ -207,7 +207,7 @@ func TestScheduler_Rollback(t *testing.T) { t.Fatal(err.Error()) } - if c.Status.State != v1alpha3.CanaryFailed { - t.Errorf("Got canary state %v wanted %v", c.Status.State, v1alpha3.CanaryFailed) + if c.Status.Phase != v1alpha3.CanaryFailed { + t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryFailed) } }