github/flagger
1
0
Fork 0
mirror of https://github.com/fluxcd/flagger.git synced 2026-04-15 06:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Scale up the deployment when canary revision changes

Browse Source
This commit is contained in:
Stefan Prodan
2018-10-02 00:24:18 +03:00
parent 26500e9ea8
commit cfc1621c06
6 changed files with 84 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+4 -6
View File
@@ -321,9 +321,6 @@ Trigger a new rollout by updating the canary image:
```bash
kubectl -n test set image deployment/podinfo-canary \
podinfod=quay.io/stefanprodan/podinfo:1.2.1
kubectl -n test scale deployment/podinfo-canary \
--replicas=1
```
Steer detects that the canary revision changed and starts a new rollout:
@@ -332,14 +329,15 @@ Steer detects that the canary revision changed and starts a new rollout:
kubectl -n test describe rollout/podinfo
Status:
Canary Revision: 16871136
Canary Revision: 19871136
Failed Checks: 0
State: finished
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Synced 3m steerer Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available
Normal Synced 3m steerer Starting rollout for podinfo.test
Normal Synced 3m steerer New revision detected podinfo-canary.test old 17211012 new 17246876
Normal Synced 3m steerer Scaling up podinfo.test
Warning Synced 3m steerer Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available
Normal Synced 3m steerer Advance rollout podinfo.test weight 10
Normal Synced 3m steerer Advance rollout podinfo.test weight 20
Normal Synced 3m steerer Advance rollout podinfo.test weight 30
artifacts/rollouts/podinfo.yaml
-3
View File
@@ -7,9 +7,6 @@
apiVersion: apps.weave.works/v1beta1
kind: Rollout
metadata:
annotations:
apps.weave.works/canary-revision: ""
apps.weave.works/status: ""
name: podinfo
namespace: test
spec:
artifacts/steerer/deployment.yaml
+1 -1
View File
@@ -22,7 +22,7 @@ spec:
serviceAccountName: steerer
containers:
- name: steerer
image: stefanprodan/steerer:0.0.1-rc.6
image: stefanprodan/steerer:0.0.1-rc.18
imagePullPolicy: Always
ports:
- name: http
docs/steerer-0.0.1.tgz
BIN
View File
Binary file not shown.
pkg/controller/deployment.go
+78 -17
View File
@@ -10,11 +10,6 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
revisionAnnotation = "apps.weave.works/canary-revision"
statusAnnotation = "apps.weave.works/status"
)
func (c *Controller) doRollouts() {
c.rollouts.Range(func(key interface{}, value interface{}) bool {
r := value.(*rolloutv1.Rollout)
@@ -39,7 +34,7 @@ func (c *Controller) advanceDeploymentRollout(name string, namespace string) {
}
// gate stage: check if canary deployment exists and is healthy
canary, ok := c.getDeployment(r, r.Spec.Canary.Name, r.Namespace)
canary, ok := c.getCanaryDeployment(r, r.Spec.Canary.Name, r.Namespace)
if !ok {
return
}
@@ -58,7 +53,7 @@ func (c *Controller) advanceDeploymentRollout(name string, namespace string) {
}
// gate stage: check if rollout should start (canary revision has changes) or continue
if ok := c.checkRolloutStatus(r, canary.ResourceVersion); !ok {
if ok := c.checkRolloutStatus(r, canary); !ok {
return
}
@@ -81,7 +76,7 @@ func (c *Controller) advanceDeploymentRollout(name string, namespace string) {
c.scaleToZeroCanary(r)
// mark rollout as failed
c.updateRolloutStatus(r, "failed")
c.updateRolloutStatus(r, "promotion-failed")
return
}
@@ -134,10 +129,9 @@ func (c *Controller) advanceDeploymentRollout(name string, namespace string) {
}
// final stage: mark rollout as finished and scale canary to zero replicas
c.updateRolloutStatus(r, "finished")
c.recordEventInfof(r, "Promotion completed! Scaling down %s.%s",
r.Name, r.Namespace, canary.GetName(), canary.Namespace)
c.recordEventInfof(r, "Scaling down %s.%s", canary.GetName(), canary.Namespace)
c.scaleToZeroCanary(r)
c.updateRolloutStatus(r, "promotion-finished")
}
}
@@ -151,12 +145,12 @@ func (c *Controller) getRollout(name string, namespace string) (*rolloutv1.Rollo
return r, true
}
func (c *Controller) checkRolloutStatus(r *rolloutv1.Rollout, canaryVersion string) bool {
func (c *Controller) checkRolloutStatus(r *rolloutv1.Rollout, canary *appsv1.Deployment) bool {
var err error
if r.Status.State == "" {
r.Status = rolloutv1.RolloutStatus{
State: "running",
CanaryRevision: canaryVersion,
CanaryRevision: canary.ResourceVersion,
FailedChecks: 0,
}
r, err = c.rolloutClient.AppsV1beta1().Rollouts(r.Namespace).Update(r)
@@ -171,10 +165,32 @@ func (c *Controller) checkRolloutStatus(r *rolloutv1.Rollout, canaryVersion stri
return true
}
if r.Status.CanaryRevision != canaryVersion {
if r.Status.State == "promotion-finished" {
c.setCanaryRevision(r, "finished")
c.logger.Infof("Promotion completed! %s.%s revision %s", r.Spec.Canary.Name, r.Namespace,
c.getDeploymentRevision(r.Spec.Canary.Name, r.Namespace))
return false
}
if r.Status.State == "promotion-failed" {
c.setCanaryRevision(r, "failed")
c.logger.Infof("Promotion failed! %s.%s revision %s", r.Spec.Canary.Name, r.Namespace,
c.getDeploymentRevision(r.Spec.Canary.Name, r.Namespace))
return false
}
if r.Status.CanaryRevision != canary.ResourceVersion {
c.recordEventInfof(r, "New revision detected %s.%s old %s new %s",
canary.GetName(), canary.Namespace, r.Status.CanaryRevision, canary.ResourceVersion)
canary.Spec.Replicas = int32p(1)
canary, err = c.kubeClient.AppsV1().Deployments(canary.Namespace).Update(canary)
if err != nil {
c.recordEventErrorf(r, "Scaling up %s.%s failed: %v", canary.GetName(), canary.Namespace, err)
return false
}
r.Status = rolloutv1.RolloutStatus{
State: "running",
CanaryRevision: canaryVersion,
CanaryRevision: canary.ResourceVersion,
FailedChecks: 0,
}
r, err = c.rolloutClient.AppsV1beta1().Rollouts(r.Namespace).Update(r)
@@ -182,7 +198,9 @@ func (c *Controller) checkRolloutStatus(r *rolloutv1.Rollout, canaryVersion stri
c.logger.Errorf("Rollout %s.%s status update failed: %v", r.Name, r.Namespace, err)
return false
}
return true
c.recordEventInfof(r, "Scaling up %s.%s", canary.GetName(), canary.Namespace)
return false
}
return false
@@ -229,6 +247,31 @@ func (c *Controller) getDeployment(r *rolloutv1.Rollout, name string, namespace
return dep, true
}
func (c *Controller) getCanaryDeployment(r *rolloutv1.Rollout, name string, namespace string) (*appsv1.Deployment, bool) {
dep, err := c.kubeClient.AppsV1().Deployments(namespace).Get(name, v1.GetOptions{})
if err != nil {
c.recordEventErrorf(r, "Deployment %s.%s not found", name, namespace)
return nil, false
}
if msg, healthy := getDeploymentStatus(dep); !healthy {
c.recordEventWarningf(r, "Halt rollout %s.%s %s", dep.GetName(), dep.Namespace, msg)
return nil, false
}
return dep, true
}
func (c *Controller) getDeploymentRevision(name string, namespace string) string {
dep, err := c.kubeClient.AppsV1().Deployments(namespace).Get(name, v1.GetOptions{})
if err != nil {
c.logger.Errorf("Deployment %s.%s not found", name, namespace)
return ""
}
return dep.ResourceVersion
}
func (c *Controller) checkDeploymentMetrics(r *rolloutv1.Rollout) bool {
for _, metric := range r.Spec.CanaryAnalysis.Metrics {
if metric.Name == "istio_requests_total" {
@@ -270,13 +313,31 @@ func (c *Controller) scaleToZeroCanary(r *rolloutv1.Rollout) {
}
//HPA https://github.com/kubernetes/kubernetes/pull/29212
canary.Spec.Replicas = int32p(0)
_, err = c.kubeClient.AppsV1().Deployments(canary.Namespace).Update(canary)
canary, err = c.kubeClient.AppsV1().Deployments(canary.Namespace).Update(canary)
if err != nil {
c.recordEventErrorf(r, "Scaling down %s.%s failed: %v", canary.GetName(), canary.Namespace, err)
return
}
}
func (c *Controller) setCanaryRevision(r *rolloutv1.Rollout, status string) {
canaryRevision := c.getDeploymentRevision(r.Spec.Canary.Name, r.Namespace)
r, ok := c.getRollout(r.Name, r.Namespace)
if !ok {
return
}
r.Status = rolloutv1.RolloutStatus{
State: status,
CanaryRevision: canaryRevision,
FailedChecks: r.Status.FailedChecks,
}
r, err := c.rolloutClient.AppsV1beta1().Rollouts(r.Namespace).Update(r)
if err != nil {
c.logger.Errorf("Rollout %s.%s status update failed: %v", r.Name, r.Namespace, err)
}
//c.logger.Infof("Rollout %s.%s status %+v", r.Spec.Canary.Name, r.Namespace, r.Status)
}
func (c *Controller) getVirtualService(r *rolloutv1.Rollout) (
vs *istiov1alpha3.VirtualService,
primary istiov1alpha3.DestinationWeight,
pkg/version/version.go
+1 -1
View File
@@ -1,4 +1,4 @@
package version
var VERSION = "0.0.1-rc.6"
var VERSION = "0.0.1-rc.18"
var REVISION = "unknown"