diff --git a/README.md b/README.md index 1012d5aa..4ceb8709 100644 --- a/README.md +++ b/README.md @@ -321,9 +321,6 @@ Trigger a new rollout by updating the canary image: ```bash kubectl -n test set image deployment/podinfo-canary \ podinfod=quay.io/stefanprodan/podinfo:1.2.1 - -kubectl -n test scale deployment/podinfo-canary \ ---replicas=1 ``` Steer detects that the canary revision changed and starts a new rollout: @@ -332,14 +329,15 @@ Steer detects that the canary revision changed and starts a new rollout: kubectl -n test describe rollout/podinfo Status: - Canary Revision: 16871136 + Canary Revision: 19871136 Failed Checks: 0 State: finished Events: Type Reason Age From Message ---- ------ ---- ---- ------- - Normal Synced 3m steerer Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available - Normal Synced 3m steerer Starting rollout for podinfo.test + Normal Synced 3m steerer New revision detected podinfo-canary.test old 17211012 new 17246876 + Normal Synced 3m steerer Scaling up podinfo.test + Warning Synced 3m steerer Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available Normal Synced 3m steerer Advance rollout podinfo.test weight 10 Normal Synced 3m steerer Advance rollout podinfo.test weight 20 Normal Synced 3m steerer Advance rollout podinfo.test weight 30 diff --git a/artifacts/rollouts/podinfo.yaml b/artifacts/rollouts/podinfo.yaml index aa8a7608..e7270a93 100644 --- a/artifacts/rollouts/podinfo.yaml +++ b/artifacts/rollouts/podinfo.yaml @@ -7,9 +7,6 @@ apiVersion: apps.weave.works/v1beta1 kind: Rollout metadata: - annotations: - apps.weave.works/canary-revision: "" - apps.weave.works/status: "" name: podinfo namespace: test spec: diff --git a/artifacts/steerer/deployment.yaml b/artifacts/steerer/deployment.yaml index f03fcdff..7062bba6 100644 --- a/artifacts/steerer/deployment.yaml +++ b/artifacts/steerer/deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: steerer containers: - name: steerer - image: stefanprodan/steerer:0.0.1-rc.6 + image: stefanprodan/steerer:0.0.1-rc.18 imagePullPolicy: Always ports: - name: http diff --git a/docs/steerer-0.0.1.tgz b/docs/steerer-0.0.1.tgz index 8195d067..d5604231 100644 Binary files a/docs/steerer-0.0.1.tgz and b/docs/steerer-0.0.1.tgz differ diff --git a/pkg/controller/deployment.go b/pkg/controller/deployment.go index a05e29ae..e40bf4e7 100644 --- a/pkg/controller/deployment.go +++ b/pkg/controller/deployment.go @@ -10,11 +10,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const ( - revisionAnnotation = "apps.weave.works/canary-revision" - statusAnnotation = "apps.weave.works/status" -) - func (c *Controller) doRollouts() { c.rollouts.Range(func(key interface{}, value interface{}) bool { r := value.(*rolloutv1.Rollout) @@ -39,7 +34,7 @@ func (c *Controller) advanceDeploymentRollout(name string, namespace string) { } // gate stage: check if canary deployment exists and is healthy - canary, ok := c.getDeployment(r, r.Spec.Canary.Name, r.Namespace) + canary, ok := c.getCanaryDeployment(r, r.Spec.Canary.Name, r.Namespace) if !ok { return } @@ -58,7 +53,7 @@ func (c *Controller) advanceDeploymentRollout(name string, namespace string) { } // gate stage: check if rollout should start (canary revision has changes) or continue - if ok := c.checkRolloutStatus(r, canary.ResourceVersion); !ok { + if ok := c.checkRolloutStatus(r, canary); !ok { return } @@ -81,7 +76,7 @@ func (c *Controller) advanceDeploymentRollout(name string, namespace string) { c.scaleToZeroCanary(r) // mark rollout as failed - c.updateRolloutStatus(r, "failed") + c.updateRolloutStatus(r, "promotion-failed") return } @@ -134,10 +129,9 @@ func (c *Controller) advanceDeploymentRollout(name string, namespace string) { } // final stage: mark rollout as finished and scale canary to zero replicas - c.updateRolloutStatus(r, "finished") - c.recordEventInfof(r, "Promotion completed! Scaling down %s.%s", - r.Name, r.Namespace, canary.GetName(), canary.Namespace) + c.recordEventInfof(r, "Scaling down %s.%s", canary.GetName(), canary.Namespace) c.scaleToZeroCanary(r) + c.updateRolloutStatus(r, "promotion-finished") } } @@ -151,12 +145,12 @@ func (c *Controller) getRollout(name string, namespace string) (*rolloutv1.Rollo return r, true } -func (c *Controller) checkRolloutStatus(r *rolloutv1.Rollout, canaryVersion string) bool { +func (c *Controller) checkRolloutStatus(r *rolloutv1.Rollout, canary *appsv1.Deployment) bool { var err error if r.Status.State == "" { r.Status = rolloutv1.RolloutStatus{ State: "running", - CanaryRevision: canaryVersion, + CanaryRevision: canary.ResourceVersion, FailedChecks: 0, } r, err = c.rolloutClient.AppsV1beta1().Rollouts(r.Namespace).Update(r) @@ -171,10 +165,32 @@ func (c *Controller) checkRolloutStatus(r *rolloutv1.Rollout, canaryVersion stri return true } - if r.Status.CanaryRevision != canaryVersion { + if r.Status.State == "promotion-finished" { + c.setCanaryRevision(r, "finished") + c.logger.Infof("Promotion completed! %s.%s revision %s", r.Spec.Canary.Name, r.Namespace, + c.getDeploymentRevision(r.Spec.Canary.Name, r.Namespace)) + return false + } + + if r.Status.State == "promotion-failed" { + c.setCanaryRevision(r, "failed") + c.logger.Infof("Promotion failed! %s.%s revision %s", r.Spec.Canary.Name, r.Namespace, + c.getDeploymentRevision(r.Spec.Canary.Name, r.Namespace)) + return false + } + + if r.Status.CanaryRevision != canary.ResourceVersion { + c.recordEventInfof(r, "New revision detected %s.%s old %s new %s", + canary.GetName(), canary.Namespace, r.Status.CanaryRevision, canary.ResourceVersion) + canary.Spec.Replicas = int32p(1) + canary, err = c.kubeClient.AppsV1().Deployments(canary.Namespace).Update(canary) + if err != nil { + c.recordEventErrorf(r, "Scaling up %s.%s failed: %v", canary.GetName(), canary.Namespace, err) + return false + } r.Status = rolloutv1.RolloutStatus{ State: "running", - CanaryRevision: canaryVersion, + CanaryRevision: canary.ResourceVersion, FailedChecks: 0, } r, err = c.rolloutClient.AppsV1beta1().Rollouts(r.Namespace).Update(r) @@ -182,7 +198,9 @@ func (c *Controller) checkRolloutStatus(r *rolloutv1.Rollout, canaryVersion stri c.logger.Errorf("Rollout %s.%s status update failed: %v", r.Name, r.Namespace, err) return false } - return true + c.recordEventInfof(r, "Scaling up %s.%s", canary.GetName(), canary.Namespace) + + return false } return false @@ -229,6 +247,31 @@ func (c *Controller) getDeployment(r *rolloutv1.Rollout, name string, namespace return dep, true } +func (c *Controller) getCanaryDeployment(r *rolloutv1.Rollout, name string, namespace string) (*appsv1.Deployment, bool) { + dep, err := c.kubeClient.AppsV1().Deployments(namespace).Get(name, v1.GetOptions{}) + if err != nil { + c.recordEventErrorf(r, "Deployment %s.%s not found", name, namespace) + return nil, false + } + + if msg, healthy := getDeploymentStatus(dep); !healthy { + c.recordEventWarningf(r, "Halt rollout %s.%s %s", dep.GetName(), dep.Namespace, msg) + return nil, false + } + + return dep, true +} + +func (c *Controller) getDeploymentRevision(name string, namespace string) string { + dep, err := c.kubeClient.AppsV1().Deployments(namespace).Get(name, v1.GetOptions{}) + if err != nil { + c.logger.Errorf("Deployment %s.%s not found", name, namespace) + return "" + } + + return dep.ResourceVersion +} + func (c *Controller) checkDeploymentMetrics(r *rolloutv1.Rollout) bool { for _, metric := range r.Spec.CanaryAnalysis.Metrics { if metric.Name == "istio_requests_total" { @@ -270,13 +313,31 @@ func (c *Controller) scaleToZeroCanary(r *rolloutv1.Rollout) { } //HPA https://github.com/kubernetes/kubernetes/pull/29212 canary.Spec.Replicas = int32p(0) - _, err = c.kubeClient.AppsV1().Deployments(canary.Namespace).Update(canary) + canary, err = c.kubeClient.AppsV1().Deployments(canary.Namespace).Update(canary) if err != nil { c.recordEventErrorf(r, "Scaling down %s.%s failed: %v", canary.GetName(), canary.Namespace, err) return } } +func (c *Controller) setCanaryRevision(r *rolloutv1.Rollout, status string) { + canaryRevision := c.getDeploymentRevision(r.Spec.Canary.Name, r.Namespace) + r, ok := c.getRollout(r.Name, r.Namespace) + if !ok { + return + } + r.Status = rolloutv1.RolloutStatus{ + State: status, + CanaryRevision: canaryRevision, + FailedChecks: r.Status.FailedChecks, + } + r, err := c.rolloutClient.AppsV1beta1().Rollouts(r.Namespace).Update(r) + if err != nil { + c.logger.Errorf("Rollout %s.%s status update failed: %v", r.Name, r.Namespace, err) + } + //c.logger.Infof("Rollout %s.%s status %+v", r.Spec.Canary.Name, r.Namespace, r.Status) +} + func (c *Controller) getVirtualService(r *rolloutv1.Rollout) ( vs *istiov1alpha3.VirtualService, primary istiov1alpha3.DestinationWeight, diff --git a/pkg/version/version.go b/pkg/version/version.go index 0cb94f9e..35a444ca 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,4 +1,4 @@ package version -var VERSION = "0.0.1-rc.6" +var VERSION = "0.0.1-rc.18" var REVISION = "unknown"