Implement status conditions

Add Promoted status condition with the following reasons: Initialized, Progressing, Succeeded, Failed
Usage: `kubectl wait canary/app --for=condition=promoted`
Fix: #184
This commit is contained in:
stefanprodan
2019-07-09 15:22:56 +03:00
parent 3e84799644
commit 438f952128
9 changed files with 197 additions and 71 deletions
+68
View File
@@ -0,0 +1,68 @@
package v1alpha3
import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// CanaryConditionType is the type of a CanaryCondition
type CanaryConditionType string
const (
// PromotedType refers to the result of the last canary analysis
PromotedType CanaryConditionType = "Promoted"
)
// CanaryCondition is a status condition for a Canary
type CanaryCondition struct {
// Type of this condition
Type CanaryConditionType `json:"type"`
// Status of this condition
Status corev1.ConditionStatus `json:"status"`
// LastUpdateTime of this condition
LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
// LastTransitionTime of this condition
LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
// Reason for the current status of this condition
Reason string `json:"reason,omitempty"`
// Message associated with this condition
Message string `json:"message,omitempty"`
}
// CanaryPhase is a label for the condition of a canary at the current time
type CanaryPhase string
const (
// CanaryPhaseInitialized means the primary deployment, hpa and ClusterIP services
// have been created along with the service mesh or ingress objects
CanaryPhaseInitialized CanaryPhase = "Initialized"
// CanaryPhaseProgressing means the canary analysis is underway
CanaryPhaseProgressing CanaryPhase = "Progressing"
// CanaryPhaseSucceeded means the canary analysis has been successful
// and the canary deployment has been promoted
CanaryPhaseSucceeded CanaryPhase = "Succeeded"
// CanaryPhaseFailed means the canary analysis failed
// and the canary deployment has been scaled to zero
CanaryPhaseFailed CanaryPhase = "Failed"
)
// CanaryStatus is used for state persistence (read-only)
type CanaryStatus struct {
Phase CanaryPhase `json:"phase"`
FailedChecks int `json:"failedChecks"`
CanaryWeight int `json:"canaryWeight"`
Iterations int `json:"iterations"`
// +optional
TrackedConfigs *map[string]string `json:"trackedConfigs,omitempty"`
// +optional
LastAppliedSpec string `json:"lastAppliedSpec,omitempty"`
// +optional
LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
// +optional
Conditions []CanaryCondition `json:"conditions,omitempty"`
}
-31
View File
@@ -85,37 +85,6 @@ type CanaryList struct {
Items []Canary `json:"items"`
}
// CanaryPhase is a label for the condition of a canary at the current time
type CanaryPhase string
const (
// CanaryInitialized means the primary deployment, hpa and ClusterIP services
// have been created along with the Istio virtual service
CanaryInitialized CanaryPhase = "Initialized"
// CanaryProgressing means the canary analysis is underway
CanaryProgressing CanaryPhase = "Progressing"
// CanarySucceeded means the canary analysis has been successful
// and the canary deployment has been promoted
CanarySucceeded CanaryPhase = "Succeeded"
// CanaryFailed means the canary analysis failed
// and the canary deployment has been scaled to zero
CanaryFailed CanaryPhase = "Failed"
)
// CanaryStatus is used for state persistence (read-only)
type CanaryStatus struct {
Phase CanaryPhase `json:"phase"`
FailedChecks int `json:"failedChecks"`
CanaryWeight int `json:"canaryWeight"`
Iterations int `json:"iterations"`
// +optional
TrackedConfigs *map[string]string `json:"trackedConfigs,omitempty"`
// +optional
LastAppliedSpec string `json:"lastAppliedSpec,omitempty"`
// +optional
LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
}
// CanaryService is used to create ClusterIP services
// and Istio Virtual Service
type CanaryService struct {
@@ -89,6 +89,24 @@ func (in *CanaryAnalysis) DeepCopy() *CanaryAnalysis {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CanaryCondition) DeepCopyInto(out *CanaryCondition) {
*out = *in
in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime)
in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryCondition.
func (in *CanaryCondition) DeepCopy() *CanaryCondition {
if in == nil {
return nil
}
out := new(CanaryCondition)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CanaryList) DeepCopyInto(out *CanaryList) {
*out = *in
@@ -250,6 +268,13 @@ func (in *CanaryStatus) DeepCopyInto(out *CanaryStatus) {
}
}
in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]CanaryCondition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
return
}
+4 -4
View File
@@ -229,7 +229,7 @@ func TestCanaryDeployer_SetState(t *testing.T) {
t.Fatal(err.Error())
}
err = mocks.deployer.SetStatusPhase(mocks.canary, v1alpha3.CanaryProgressing)
err = mocks.deployer.SetStatusPhase(mocks.canary, v1alpha3.CanaryPhaseProgressing)
if err != nil {
t.Fatal(err.Error())
}
@@ -239,8 +239,8 @@ func TestCanaryDeployer_SetState(t *testing.T) {
t.Fatal(err.Error())
}
if res.Status.Phase != v1alpha3.CanaryProgressing {
t.Errorf("Got %v wanted %v", res.Status.Phase, v1alpha3.CanaryProgressing)
if res.Status.Phase != v1alpha3.CanaryPhaseProgressing {
t.Errorf("Got %v wanted %v", res.Status.Phase, v1alpha3.CanaryPhaseProgressing)
}
}
@@ -252,7 +252,7 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) {
}
status := v1alpha3.CanaryStatus{
Phase: v1alpha3.CanaryProgressing,
Phase: v1alpha3.CanaryPhaseProgressing,
FailedChecks: 2,
}
err = mocks.deployer.SyncStatus(mocks.canary, status)
+65 -1
View File
@@ -5,6 +5,7 @@ import (
"github.com/mitchellh/hashstructure"
flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -38,6 +39,10 @@ func (c *Deployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatu
cdCopy.Status.LastTransitionTime = metav1.Now()
cdCopy.Status.TrackedConfigs = configs
if ok, conditions := c.makeStatusConditions(cd.Status, status.Phase); ok {
cdCopy.Status.Conditions = conditions
}
cd, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
if err != nil {
return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
@@ -90,14 +95,73 @@ func (c *Deployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPh
cdCopy.Status.Phase = phase
cdCopy.Status.LastTransitionTime = metav1.Now()
if phase != flaggerv1.CanaryProgressing {
if phase != flaggerv1.CanaryPhaseProgressing {
cdCopy.Status.CanaryWeight = 0
cdCopy.Status.Iterations = 0
}
if ok, conditions := c.makeStatusConditions(cdCopy.Status, phase); ok {
cdCopy.Status.Conditions = conditions
}
cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
if err != nil {
return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
}
return nil
}
// GetStatusCondition returns a condition based on type
func (c *Deployer) getStatusCondition(status flaggerv1.CanaryStatus, conditionType flaggerv1.CanaryConditionType) *flaggerv1.CanaryCondition {
for i := range status.Conditions {
c := status.Conditions[i]
if c.Type == conditionType {
return &c
}
}
return nil
}
// MakeStatusCondition updates the canary status conditions based on canary phase
func (c *Deployer) makeStatusConditions(canaryStatus flaggerv1.CanaryStatus,
phase flaggerv1.CanaryPhase) (bool, []flaggerv1.CanaryCondition) {
currentCondition := c.getStatusCondition(canaryStatus, flaggerv1.PromotedType)
message := "New deployment detected, starting initialization."
status := corev1.ConditionUnknown
switch phase {
case flaggerv1.CanaryPhaseProgressing:
status = corev1.ConditionUnknown
message = "New revision detected, starting canary analysis."
case flaggerv1.CanaryPhaseInitialized:
status = corev1.ConditionTrue
message = "New deployment detected, initialization completed."
case flaggerv1.CanaryPhaseSucceeded:
status = corev1.ConditionTrue
message = "Canary analysis completed successfully, promotion finished."
case flaggerv1.CanaryPhaseFailed:
status = corev1.ConditionFalse
message = "Canary analysis failed, deployment scaled to zero."
}
newCondition := &flaggerv1.CanaryCondition{
Type: flaggerv1.PromotedType,
Status: status,
LastUpdateTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Message: message,
Reason: string(phase),
}
if currentCondition != nil &&
currentCondition.Status == newCondition.Status &&
currentCondition.Reason == newCondition.Reason {
return false, nil
}
if currentCondition != nil && currentCondition.Status == newCondition.Status {
newCondition.LastTransitionTime = currentCondition.LastTransitionTime
}
return true, []flaggerv1.CanaryCondition{*newCondition}
}
+22 -22
View File
@@ -178,7 +178,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
// reset status
status := flaggerv1.CanaryStatus{
Phase: flaggerv1.CanaryProgressing,
Phase: flaggerv1.CanaryPhaseProgressing,
CanaryWeight: 0,
FailedChecks: 0,
Iterations: 0,
@@ -210,7 +210,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
}
// check if the number of failed checks reached the threshold
if cd.Status.Phase == flaggerv1.CanaryProgressing &&
if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing &&
(!retriable || cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold) {
if cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold {
@@ -246,13 +246,13 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
}
// mark canary as failed
if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed, CanaryWeight: 0}); err != nil {
if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseFailed, CanaryWeight: 0}); err != nil {
c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err)
return
}
c.recorder.SetStatus(cd, flaggerv1.CanaryFailed)
c.runPostRolloutHooks(cd, flaggerv1.CanaryFailed)
c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseFailed)
c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseFailed)
return
}
@@ -331,12 +331,12 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
}
// update status phase
if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil {
if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
c.recorder.SetStatus(cd, flaggerv1.CanarySucceeded)
c.runPostRolloutHooks(cd, flaggerv1.CanarySucceeded)
c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded)
c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded)
c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.",
false, false)
return
@@ -398,12 +398,12 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
}
// update status phase
if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil {
if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
c.recorder.SetStatus(cd, flaggerv1.CanarySucceeded)
c.runPostRolloutHooks(cd, flaggerv1.CanarySucceeded)
c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded)
c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded)
c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.",
false, false)
}
@@ -438,13 +438,13 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router.
}
// update status phase
if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil {
if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil {
c.recordEventWarningf(cd, "%v", err)
return false
}
// notify
c.recorder.SetStatus(cd, flaggerv1.CanarySucceeded)
c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded)
c.recordEventInfof(cd, "Promotion completed! Canary analysis was skipped for %s.%s",
cd.Spec.TargetRef.Name, cd.Namespace)
c.sendNotification(cd, "Canary analysis was skipped, promotion finished.",
@@ -454,7 +454,7 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router.
}
func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) {
if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryProgressing {
if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryPhaseProgressing {
return true, nil
}
@@ -477,16 +477,16 @@ func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) {
func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool) bool {
c.recorder.SetStatus(cd, cd.Status.Phase)
if cd.Status.Phase == flaggerv1.CanaryProgressing {
if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing {
return true
}
if cd.Status.Phase == "" {
if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryInitialized}); err != nil {
if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseInitialized}); err != nil {
c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err)
return false
}
c.recorder.SetStatus(cd, flaggerv1.CanaryInitialized)
c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseInitialized)
c.recordEventInfof(cd, "Initialization done! %s.%s", cd.Name, cd.Namespace)
c.sendNotification(cd, "New deployment detected, initialization completed.",
true, false)
@@ -501,18 +501,18 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool)
c.recordEventErrorf(cd, "%v", err)
return false
}
if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryProgressing}); err != nil {
if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseProgressing}); err != nil {
c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err)
return false
}
c.recorder.SetStatus(cd, flaggerv1.CanaryProgressing)
c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseProgressing)
return false
}
return false
}
func (c *Controller) hasCanaryRevisionChanged(cd *flaggerv1.Canary) bool {
if cd.Status.Phase == flaggerv1.CanaryProgressing {
if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing {
if diff, _ := c.deployer.HasDeploymentChanged(cd); diff {
return true
}
@@ -526,7 +526,7 @@ func (c *Controller) hasCanaryRevisionChanged(cd *flaggerv1.Canary) bool {
func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool {
for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
if webhook.Type == flaggerv1.PreRolloutHook {
err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryProgressing, webhook)
err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
if err != nil {
c.recordEventWarningf(canary, "Halt %s.%s advancement pre-rollout check %s failed %v",
canary.Name, canary.Namespace, webhook.Name, err)
@@ -558,7 +558,7 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
// run external checks
for _, webhook := range r.Spec.CanaryAnalysis.Webhooks {
if webhook.Type == "" || webhook.Type == flaggerv1.RolloutHook {
err := CallWebhook(r.Name, r.Namespace, flaggerv1.CanaryProgressing, webhook)
err := CallWebhook(r.Name, r.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
if err != nil {
c.recordEventWarningf(r, "Halt %s.%s advancement external check %s failed %v",
r.Name, r.Namespace, webhook.Name, err)
+9 -9
View File
@@ -47,7 +47,7 @@ func TestScheduler_Rollback(t *testing.T) {
mocks.ctrl.advanceCanary("podinfo", "default", true)
// update failed checks to max
err := mocks.deployer.SyncStatus(mocks.canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryProgressing, FailedChecks: 11})
err := mocks.deployer.SyncStatus(mocks.canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryPhaseProgressing, FailedChecks: 11})
if err != nil {
t.Fatal(err.Error())
}
@@ -60,8 +60,8 @@ func TestScheduler_Rollback(t *testing.T) {
t.Fatal(err.Error())
}
if c.Status.Phase != v1alpha3.CanaryFailed {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryFailed)
if c.Status.Phase != v1alpha3.CanaryPhaseFailed {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseFailed)
}
}
@@ -101,8 +101,8 @@ func TestScheduler_SkipAnalysis(t *testing.T) {
t.Errorf("Got skip analysis %v wanted %v", c.Spec.SkipAnalysis, true)
}
if c.Status.Phase != v1alpha3.CanarySucceeded {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded)
if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded)
}
}
@@ -255,8 +255,8 @@ func TestScheduler_Promotion(t *testing.T) {
t.Fatal(err.Error())
}
if c.Status.Phase != v1alpha3.CanarySucceeded {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded)
if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded)
}
}
@@ -326,8 +326,8 @@ func TestScheduler_ABTesting(t *testing.T) {
t.Fatal(err.Error())
}
if c.Status.Phase != v1alpha3.CanarySucceeded {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded)
if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded)
}
}
+2 -2
View File
@@ -19,7 +19,7 @@ func TestCallWebhook(t *testing.T) {
Metadata: &map[string]string{"key1": "val1"},
}
err := CallWebhook("podinfo", "default", flaggerv1.CanaryProgressing, hook)
err := CallWebhook("podinfo", "default", flaggerv1.CanaryPhaseProgressing, hook)
if err != nil {
t.Fatal(err.Error())
}
@@ -35,7 +35,7 @@ func TestCallWebhook_StatusCode(t *testing.T) {
URL: ts.URL,
}
err := CallWebhook("podinfo", "default", flaggerv1.CanaryProgressing, hook)
err := CallWebhook("podinfo", "default", flaggerv1.CanaryPhaseProgressing, hook)
if err == nil {
t.Errorf("Got no error wanted %v", http.StatusInternalServerError)
}
+2 -2
View File
@@ -87,9 +87,9 @@ func (cr *Recorder) SetTotal(namespace string, total int) {
func (cr *Recorder) SetStatus(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) {
status := 1
switch phase {
case flaggerv1.CanaryProgressing:
case flaggerv1.CanaryPhaseProgressing:
status = 0
case flaggerv1.CanaryFailed:
case flaggerv1.CanaryPhaseFailed:
status = 2
default:
status = 1